26 #ifdef ENABLE_IMPORT_PARQUET
39 std::string data_wrapper_type;
44 #ifdef ENABLE_IMPORT_PARQUET
51 return data_wrapper_type;
55 namespace foreign_storage {
56 std::tuple<std::unique_ptr<foreign_storage::ForeignServer>,
57 std::unique_ptr<foreign_storage::UserMapping>,
58 std::unique_ptr<foreign_storage::ForeignTable>>
63 const int32_t user_id) {
65 db_id, user_id, copy_from_source, copy_params);
72 db_id, user_id, copy_from_source, copy_params, server.get());
75 user_mapping->validate(server.get());
80 db_id, table, copy_from_source, copy_params, server.get());
83 foreign_table->validateOptionValues();
85 return {std::move(server), std::move(user_mapping), std::move(foreign_table)};
88 std::tuple<std::unique_ptr<foreign_storage::ForeignServer>,
89 std::unique_ptr<foreign_storage::UserMapping>,
90 std::unique_ptr<foreign_storage::ForeignTable>>
103 #ifdef ENABLE_IMPORT_PARQUET
112 namespace foreign_storage {
116 throw std::runtime_error{
"Regex parser options must contain a line regex."};
122 #ifdef ENABLE_IMPORT_PARQUET
130 return value ?
"TRUE" :
"FALSE";
142 return std::make_unique<CsvDataWrapper>(
143 db_id, foreign_table, user_mapping,
true);
145 return std::make_unique<RegexParserDataWrapper>(
146 db_id, foreign_table, user_mapping,
true);
148 #ifdef ENABLE_IMPORT_PARQUET
150 return std::make_unique<ParquetDataWrapper>(
151 db_id, foreign_table,
false);
159 const std::string& data_wrapper_type,
163 #ifdef ENABLE_IMPORT_PARQUET
166 return std::make_unique<ParquetImporter>(db_id, foreign_table, user_mapping);
172 std::unique_ptr<UserMapping>
176 const std::string& file_path,
185 const std::string& file_path,
189 auto foreign_server = std::make_unique<foreign_storage::ForeignServer>();
191 foreign_server->id = -1;
192 foreign_server->user_id = user_id;
197 #ifdef ENABLE_IMPORT_PARQUET
204 foreign_server->name =
"import_proxy_server";
207 throw std::runtime_error(
"ODBC storage not supported");
209 throw std::runtime_error(
"AWS storage not supported");
215 return foreign_server;
221 switch (has_header) {
238 const std::string& copy_from_source,
244 auto foreign_table = std::make_unique<ForeignTable>();
250 foreign_table->foreign_server = server;
287 throw std::runtime_error(
"ODBC storage not supported");
289 throw std::runtime_error(
"AWS storage not supported");
291 foreign_table->options[
"FILE_PATH"] = copy_from_source;
307 const std::array<char, 3> array_marker{
315 throw std::runtime_error(
316 "geo_explode_collections is not yet supported for FSI CSV import");
330 foreign_table->initializeOptions();
331 return foreign_table;
335 const std::string& data_wrapper_type,
338 std::unique_ptr<ForeignDataWrapper> data_wrapper;
343 data_wrapper = std::make_unique<CsvDataWrapper>(db_id, foreign_table);
345 #ifdef ENABLE_IMPORT_PARQUET
347 data_wrapper = std::make_unique<ParquetDataWrapper>(db_id, foreign_table);
350 data_wrapper = std::make_unique<RegexParserDataWrapper>(db_id, foreign_table);
352 data_wrapper = std::make_unique<InternalCatalogDataWrapper>(db_id, foreign_table);
354 data_wrapper = std::make_unique<InternalMemoryStatsDataWrapper>(db_id, foreign_table);
357 std::make_unique<InternalStorageStatsDataWrapper>(db_id, foreign_table);
359 data_wrapper = std::make_unique<InternalLogsDataWrapper>(db_id, foreign_table);
361 throw std::runtime_error(
"Unsupported data wrapper");
367 const std::string& data_wrapper_type,
369 bool is_s3_select_wrapper{
false};
370 std::string data_wrapper_type_key{data_wrapper_type};
371 constexpr
const char* S3_SELECT_WRAPPER_KEY =
"CSV_S3_SELECT";
374 is_s3_select_wrapper =
true;
375 data_wrapper_type_key = S3_SELECT_WRAPPER_KEY;
381 if (is_s3_select_wrapper) {
385 std::make_unique<CsvDataWrapper>();
387 #ifdef ENABLE_IMPORT_PARQUET
390 std::make_unique<ParquetDataWrapper>();
394 std::make_unique<RegexParserDataWrapper>();
397 std::make_unique<InternalCatalogDataWrapper>();
400 std::make_unique<InternalMemoryStatsDataWrapper>();
403 std::make_unique<InternalStorageStatsDataWrapper>();
406 std::make_unique<InternalLogsDataWrapper>();
417 const std::string& data_wrapper_type) {
419 if (std::find(supported_wrapper_types.begin(),
420 supported_wrapper_types.end(),
421 data_wrapper_type) == supported_wrapper_types.end()) {
422 std::vector<std::string_view> user_facing_wrapper_types;
423 for (
const auto&
type : supported_wrapper_types) {
425 user_facing_wrapper_types.emplace_back(
type);
428 throw std::runtime_error{
"Invalid data wrapper type \"" + data_wrapper_type +
429 "\". Data wrapper type must be one of the following: " +
430 join(user_facing_wrapper_types,
", ") +
"."};
434 std::map<std::string, std::unique_ptr<ForeignDataWrapper>>
bool contains(const T &container, const U &element)
static const std::string GEO_EXPLODE_COLLECTIONS_KEY
static const ForeignDataWrapper & createForValidation(const std::string &data_wrapper_type, const ForeignTable *foreign_table=nullptr)
bool is_valid_data_wrapper(const std::string &data_wrapper_type)
static std::unique_ptr< ForeignDataWrapper > createForImport(const std::string &data_wrapper_type, const int db_id, const ForeignTable *foreign_table, const UserMapping *user_mapping)
static const std::string BUFFER_SIZE_KEY
static constexpr char const * REGEX_PARSER
static const std::string REGEX_PATH_FILTER_KEY
static std::unique_ptr< ForeignDataWrapper > create(const std::string &data_wrapper_type, const int db_id, const ForeignTable *foreign_table)
shared utility for globbing files, paths can be specified as either a single file, directory or wildcards
static const std::string HEADER_KEY
static const std::string TRIM_SPACES_KEY
static const std::string ARRAY_MARKER_KEY
bool is_s3_uri(const std::string &file_path)
static const std::string LOCAL_FILE_STORAGE_TYPE
static constexpr char const * INTERNAL_STORAGE_STATS
bool geo_explode_collections
static const std::string LINE_REGEX_KEY
static constexpr std::array< std::string_view, 7 > supported_data_wrapper_types
static const std::string NULLS_KEY
static constexpr std::array< char const *, 4 > INTERNAL_DATA_WRAPPERS
static bool validateAndGetIsS3Select(const ForeignTable *foreign_table)
void set_header_option(OptionsMap &options, const import_export::ImportHeaderRow &has_header)
static constexpr char const * INTERNAL_CATALOG
std::tuple< std::unique_ptr< foreign_storage::ForeignServer >, std::unique_ptr< foreign_storage::UserMapping >, std::unique_ptr< foreign_storage::ForeignTable > > create_proxy_fsi_objects(const std::string ©_from_source, const import_export::CopyParams ©_params, const int db_id, const TableDescriptor *table, const int32_t user_id)
Create proxy fsi objects for use outside FSI.
static std::map< std::string, std::unique_ptr< ForeignDataWrapper > > validation_data_wrappers_
static const std::string FILE_SORT_ORDER_BY_KEY
ImportHeaderRow has_header
static const std::string SOURCE_SRID_KEY
bool geo_assign_render_groups
std::optional< std::string > regex_path_filter
static const std::string LONLAT_KEY
static std::unique_ptr< ForeignDataWrapper > createForGeneralImport(const import_export::CopyParams ©_params, const int db_id, const ForeignTable *foreign_table, const UserMapping *user_mapping)
static SysCatalog & instance()
static const std::string STORAGE_TYPE_KEY
std::string bool_to_option_value(const bool value)
static void validateDataWrapperType(const std::string &data_wrapper_type)
std::string get_data_wrapper_type(const import_export::CopyParams ©_params)
static const std::string LINE_DELIMITER_KEY
import_export::SourceType source_type
bool is_valid_source_type(const import_export::CopyParams ©_params)
size_t num_import_threads(const int32_t copy_params_threads)
static const std::string DELIMITER_KEY
std::shared_ptr< Catalog > getCatalog(const std::string &dbName)
void validate_regex_parser_options(const import_export::CopyParams ©_params)
static const std::string ARRAY_DELIMITER_KEY
std::string line_start_regex
static constexpr char const * INTERNAL_MEMORY_STATS
static std::unique_ptr< ForeignServer > createForeignServerProxy(const int db_id, const int user_id, const std::string &file_path, const import_export::CopyParams ©_params)
static std::unique_ptr< ForeignTable > createForeignTableProxy(const int db_id, const TableDescriptor *table, const std::string &file_path, const import_export::CopyParams ©_params, const ForeignServer *server)
static const std::string THREADS_KEY
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
static const std::string LINE_START_REGEX_KEY
static const std::string FILE_SORT_REGEX_KEY
static const std::string QUOTED_KEY
static const std::string GEO_ASSIGN_RENDER_GROUPS_KEY
static const std::string QUOTE_KEY
static constexpr char const * INTERNAL_LOGS
std::optional< std::string > file_sort_order_by
static const std::string ESCAPE_KEY
static constexpr char const * PARQUET
std::optional< std::string > file_sort_regex
static std::unique_ptr< UserMapping > createUserMappingProxyIfApplicable(const int db_id, const int user_id, const std::string &file_path, const import_export::CopyParams ©_params, const ForeignServer *server)