OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::ForeignDataWrapperFactory Class Reference

#include <ForeignDataWrapperFactory.h>

Static Public Member Functions

static std::unique_ptr
< ForeignDataWrapper
create (const std::string &data_wrapper_type, const int db_id, const ForeignTable *foreign_table)
 
static std::unique_ptr
< UserMapping
createUserMappingProxyIfApplicable (const int db_id, const int user_id, const std::string &file_path, const import_export::CopyParams &copy_params, const ForeignServer *server)
 
static std::unique_ptr
< ForeignServer
createForeignServerProxy (const int db_id, const int user_id, const std::string &file_path, const import_export::CopyParams &copy_params)
 
static std::unique_ptr
< ForeignTable
createForeignTableProxy (const int db_id, const TableDescriptor *table, const std::string &file_path, const import_export::CopyParams &copy_params, const ForeignServer *server)
 
static std::unique_ptr
< ForeignDataWrapper
createForImport (const std::string &data_wrapper_type, const int db_id, const ForeignTable *foreign_table, const UserMapping *user_mapping)
 
static const ForeignDataWrappercreateForValidation (const std::string &data_wrapper_type, const ForeignTable *foreign_table=nullptr)
 
static void validateDataWrapperType (const std::string &data_wrapper_type)
 

Static Private Attributes

static std::map< std::string,
std::unique_ptr
< ForeignDataWrapper > > 
validation_data_wrappers_
 

Detailed Description

Definition at line 47 of file ForeignDataWrapperFactory.h.

Member Function Documentation

std::unique_ptr< ForeignDataWrapper > foreign_storage::ForeignDataWrapperFactory::create ( const std::string &  data_wrapper_type,
const int  db_id,
const ForeignTable foreign_table 
)
static

Creates an instance of a ForeignDataWrapper for the given data wrapper type using provided database and foreign table details.

Definition at line 129 of file ForeignDataWrapperFactory.cpp.

References foreign_storage::DataWrapperType::CSV, foreign_storage::DataWrapperType::INTERNAL_CATALOG, foreign_storage::DataWrapperType::INTERNAL_MEMORY_STATS, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, UNREACHABLE, and foreign_storage::CsvDataWrapper::validateAndGetIsS3Select().

132  {
133  std::unique_ptr<ForeignDataWrapper> data_wrapper;
134  if (data_wrapper_type == DataWrapperType::CSV) {
135  if (CsvDataWrapper::validateAndGetIsS3Select(foreign_table)) {
136  UNREACHABLE();
137  } else {
138  data_wrapper = std::make_unique<CsvDataWrapper>(db_id, foreign_table);
139  }
140 #ifdef ENABLE_IMPORT_PARQUET
141  } else if (data_wrapper_type == DataWrapperType::PARQUET) {
142  data_wrapper = std::make_unique<ParquetDataWrapper>(db_id, foreign_table);
143 #endif
144  } else if (data_wrapper_type == DataWrapperType::REGEX_PARSER) {
145  data_wrapper = std::make_unique<RegexParserDataWrapper>(db_id, foreign_table);
146  } else if (data_wrapper_type == DataWrapperType::INTERNAL_CATALOG) {
147  data_wrapper = std::make_unique<InternalCatalogDataWrapper>(db_id, foreign_table);
148  } else if (data_wrapper_type == DataWrapperType::INTERNAL_MEMORY_STATS) {
149  data_wrapper = std::make_unique<InternalMemoryStatsDataWrapper>(db_id, foreign_table);
150  } else {
151  throw std::runtime_error("Unsupported data wrapper");
152  }
153  return data_wrapper;
154 }
static constexpr char const * REGEX_PARSER
#define UNREACHABLE()
Definition: Logger.h:253
static bool validateAndGetIsS3Select(const ForeignTable *foreign_table)
static constexpr char const * INTERNAL_CATALOG
static constexpr char const * INTERNAL_MEMORY_STATS
static constexpr char const * CSV
static constexpr char const * PARQUET

+ Here is the call graph for this function:

std::unique_ptr< ForeignServer > foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy ( const int  db_id,
const int  user_id,
const std::string &  file_path,
const import_export::CopyParams copy_params 
)
static

Definition at line 66 of file ForeignDataWrapperFactory.cpp.

References CHECK, import_export::CopyParams::file_type, anonymous_namespace{ForeignDataWrapperFactory.cpp}::is_s3_uri(), foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, foreign_storage::DataWrapperType::PARQUET, foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY, and UNREACHABLE.

Referenced by import_export::ForeignDataImporter::import().

70  {
71 // only supported for parquet import path currently
72 #ifdef ENABLE_IMPORT_PARQUET
73  CHECK(copy_params.file_type == import_export::FileType::PARQUET);
74 #else
75  UNREACHABLE() << "Unexpected method call for non-Parquet import";
76 #endif
77 
78  auto foreign_server = std::make_unique<foreign_storage::ForeignServer>();
79 
80  foreign_server->id = -1;
81  foreign_server->user_id = user_id;
82  foreign_server->data_wrapper_type = DataWrapperType::PARQUET;
83  foreign_server->name = "import_proxy_server";
84 
85  bool is_aws_s3_storage_type = is_s3_uri(file_path);
86  if (is_aws_s3_storage_type) {
87  throw std::runtime_error("AWS storage not supported");
88  } else {
89  foreign_server->options[AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY] =
91  }
92 
93  return foreign_server;
94 }
#define UNREACHABLE()
Definition: Logger.h:253
#define CHECK(condition)
Definition: Logger.h:209
static constexpr char const * PARQUET

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::unique_ptr< ForeignTable > foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy ( const int  db_id,
const TableDescriptor table,
const std::string &  file_path,
const import_export::CopyParams copy_params,
const ForeignServer server 
)
static

Definition at line 96 of file ForeignDataWrapperFactory.cpp.

References CHECK, import_export::CopyParams::file_type, Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), anonymous_namespace{ForeignDataWrapperFactory.cpp}::is_s3_uri(), and UNREACHABLE.

Referenced by import_export::ForeignDataImporter::import().

101  {
102 // only supported for parquet import path currently
103 #ifdef ENABLE_IMPORT_PARQUET
104  CHECK(copy_params.file_type == import_export::FileType::PARQUET);
105 #else
106  UNREACHABLE() << "Unexpected method call for non-Parquet import";
107 #endif
108 
109  auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
110  auto foreign_table = std::make_unique<ForeignTable>();
111 
112  *static_cast<TableDescriptor*>(foreign_table.get()) =
113  *table; // copy table related values
114 
115  CHECK(server);
116  foreign_table->foreign_server = server;
117 
118  bool is_aws_s3_storage_type = is_s3_uri(file_path);
119  if (is_aws_s3_storage_type) {
120  throw std::runtime_error("AWS storage not supported");
121  } else {
122  foreign_table->options["FILE_PATH"] = file_path;
123  }
124 
125  foreign_table->initializeOptions();
126  return foreign_table;
127 }
#define UNREACHABLE()
Definition: Logger.h:253
static SysCatalog & instance()
Definition: SysCatalog.h:325
std::shared_ptr< Catalog > getCatalog(const std::string &dbName)
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::unique_ptr< ForeignDataWrapper > foreign_storage::ForeignDataWrapperFactory::createForImport ( const std::string &  data_wrapper_type,
const int  db_id,
const ForeignTable foreign_table,
const UserMapping user_mapping 
)
static

Create for the import use-case.

Definition at line 42 of file ForeignDataWrapperFactory.cpp.

References CHECK, and foreign_storage::DataWrapperType::PARQUET.

Referenced by import_export::ForeignDataImporter::import().

46  {
47 #ifdef ENABLE_IMPORT_PARQUET
48  // only supported for parquet import path currently
49  CHECK(data_wrapper_type == DataWrapperType::PARQUET);
50  return std::make_unique<ParquetImporter>(db_id, foreign_table, user_mapping);
51 #else
52  return {};
53 #endif
54 }
#define CHECK(condition)
Definition: Logger.h:209
static constexpr char const * PARQUET

+ Here is the caller graph for this function:

const ForeignDataWrapper & foreign_storage::ForeignDataWrapperFactory::createForValidation ( const std::string &  data_wrapper_type,
const ForeignTable foreign_table = nullptr 
)
static

Creates an instance (or gets an existing instance) of an immutable ForeignDataWrapper to be used for validation purposes. Returned instance should not be used for any stateful operations, such as fetching foreign table data/metadata.

Definition at line 156 of file ForeignDataWrapperFactory.cpp.

References CHECK, foreign_storage::DataWrapperType::CSV, foreign_storage::DataWrapperType::INTERNAL_CATALOG, foreign_storage::DataWrapperType::INTERNAL_MEMORY_STATS, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, UNREACHABLE, foreign_storage::CsvDataWrapper::validateAndGetIsS3Select(), and validation_data_wrappers_.

Referenced by foreign_storage::ForeignServer::validateStorageParameters().

158  {
159  bool is_s3_select_wrapper{false};
160  std::string data_wrapper_type_key{data_wrapper_type};
161  constexpr const char* S3_SELECT_WRAPPER_KEY = "CSV_S3_SELECT";
162  if (foreign_table && data_wrapper_type == DataWrapperType::CSV &&
164  is_s3_select_wrapper = true;
165  data_wrapper_type_key = S3_SELECT_WRAPPER_KEY;
166  }
167 
168  if (validation_data_wrappers_.find(data_wrapper_type_key) ==
170  if (data_wrapper_type == DataWrapperType::CSV) {
171  if (is_s3_select_wrapper) {
172  UNREACHABLE();
173  } else {
174  validation_data_wrappers_[data_wrapper_type_key] =
175  std::make_unique<CsvDataWrapper>();
176  }
177 #ifdef ENABLE_IMPORT_PARQUET
178  } else if (data_wrapper_type == DataWrapperType::PARQUET) {
179  validation_data_wrappers_[data_wrapper_type_key] =
180  std::make_unique<ParquetDataWrapper>();
181 #endif
182  } else if (data_wrapper_type == DataWrapperType::REGEX_PARSER) {
183  validation_data_wrappers_[data_wrapper_type_key] =
184  std::make_unique<RegexParserDataWrapper>();
185  } else if (data_wrapper_type == DataWrapperType::INTERNAL_CATALOG) {
186  validation_data_wrappers_[data_wrapper_type_key] =
187  std::make_unique<InternalCatalogDataWrapper>();
188  } else if (data_wrapper_type == DataWrapperType::INTERNAL_MEMORY_STATS) {
189  validation_data_wrappers_[data_wrapper_type_key] =
190  std::make_unique<InternalMemoryStatsDataWrapper>();
191  } else {
192  UNREACHABLE();
193  }
194  }
195  CHECK(validation_data_wrappers_.find(data_wrapper_type_key) !=
197  return *validation_data_wrappers_[data_wrapper_type_key];
198 }
static constexpr char const * REGEX_PARSER
#define UNREACHABLE()
Definition: Logger.h:253
static bool validateAndGetIsS3Select(const ForeignTable *foreign_table)
static constexpr char const * INTERNAL_CATALOG
static std::map< std::string, std::unique_ptr< ForeignDataWrapper > > validation_data_wrappers_
static constexpr char const * INTERNAL_MEMORY_STATS
#define CHECK(condition)
Definition: Logger.h:209
static constexpr char const * CSV
static constexpr char const * PARQUET

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::unique_ptr< UserMapping > foreign_storage::ForeignDataWrapperFactory::createUserMappingProxyIfApplicable ( const int  db_id,
const int  user_id,
const std::string &  file_path,
const import_export::CopyParams copy_params,
const ForeignServer server 
)
static

Definition at line 57 of file ForeignDataWrapperFactory.cpp.

Referenced by import_export::ForeignDataImporter::import().

62  {
63  return {};
64 }

+ Here is the caller graph for this function:

void foreign_storage::ForeignDataWrapperFactory::validateDataWrapperType ( const std::string &  data_wrapper_type)
static

Checks that the given data wrapper type is valid.

Definition at line 200 of file ForeignDataWrapperFactory.cpp.

References shared::contains(), foreign_storage::DataWrapperType::INTERNAL_DATA_WRAPPERS, join(), foreign_storage::DataWrapperType::supported_data_wrapper_types, and run_benchmark_import::type.

Referenced by foreign_storage::ForeignServer::validate().

201  {
202  const auto& supported_wrapper_types = DataWrapperType::supported_data_wrapper_types;
203  if (std::find(supported_wrapper_types.begin(),
204  supported_wrapper_types.end(),
205  data_wrapper_type) == supported_wrapper_types.end()) {
206  std::vector<std::string_view> user_facing_wrapper_types;
207  for (const auto& type : supported_wrapper_types) {
209  user_facing_wrapper_types.emplace_back(type);
210  }
211  }
212  throw std::runtime_error{"Invalid data wrapper type \"" + data_wrapper_type +
213  "\". Data wrapper type must be one of the following: " +
214  join(user_facing_wrapper_types, ", ") + "."};
215  }
216 }
bool contains(const T &container, const U &element)
Definition: misc.h:188
static constexpr std::array< std::string_view, 5 > supported_data_wrapper_types
std::string join(T const &container, std::string const &delim)
static constexpr std::array< char const *, 2 > INTERNAL_DATA_WRAPPERS

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::map< std::string, std::unique_ptr< ForeignDataWrapper > > foreign_storage::ForeignDataWrapperFactory::validation_data_wrappers_
staticprivate

Definition at line 102 of file ForeignDataWrapperFactory.h.

Referenced by createForValidation().


The documentation for this class was generated from the following files: