OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::AbstractFileStorageDataWrapper Class Reference

#include <AbstractFileStorageDataWrapper.h>

+ Inheritance diagram for foreign_storage::AbstractFileStorageDataWrapper:
+ Collaboration diagram for foreign_storage::AbstractFileStorageDataWrapper:

Public Member Functions

 AbstractFileStorageDataWrapper ()
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 
getSupportedTableOptions () const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 
getSupportedUserMappingOptions () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector)=0
 
virtual void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers)=0
 
virtual std::string getSerializedDataWrapper () const =0
 
virtual void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata)=0
 
virtual bool isRestored () const =0
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 
virtual ParallelismLevel getCachedParallelismLevel () const
 
virtual ParallelismLevel getNonCachedParallelismLevel () const
 

Static Public Attributes

static const std::string STORAGE_TYPE_KEY = "STORAGE_TYPE"
 
static const std::string BASE_PATH_KEY = "BASE_PATH"
 
static const std::string FILE_PATH_KEY = "FILE_PATH"
 
static const std::string REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
 
static const std::string LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"
 
static const std::string S3_STORAGE_TYPE = "AWS_S3"
 
static const std::string FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
 
static const std::string FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
 
static const std::array
< std::string, 1 > 
supported_storage_types
 

Static Protected Member Functions

static std::string getFullFilePath (const ForeignTable *foreign_table)
 Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...
 

Static Private Member Functions

static void validateFilePath (const ForeignTable *foreign_table)
 
static void validateFilePathOptionKey (const ForeignTable *foreign_table)
 

Static Private Attributes

static const std::set
< std::string_view > 
supported_table_options_
 
static const std::set
< std::string_view > 
supported_server_options_
 
static const std::set
< std::string_view > 
supported_user_mapping_options_ {}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 

Detailed Description

Definition at line 8 of file AbstractFileStorageDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::AbstractFileStorageDataWrapper::AbstractFileStorageDataWrapper ( )

Definition at line 12 of file AbstractFileStorageDataWrapper.cpp.

12 {}

Member Function Documentation

std::string foreign_storage::AbstractFileStorageDataWrapper::getFullFilePath ( const ForeignTable foreign_table)
staticprotected

Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options.

Definition at line 92 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::append_file_path(), BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, and STORAGE_TYPE_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::ParquetImporter::getAllFilePaths(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), foreign_storage::AbstractTextFileDataWrapper::populateChunks(), foreign_storage::AbstractTextFileDataWrapper::restoreDataWrapperInternals(), and validateFilePath().

93  {
94  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
95  std::optional<std::string> base_path{};
96  auto foreign_server = foreign_table->foreign_server;
97  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
98  CHECK(storage_type);
99 
100  const std::string separator{boost::filesystem::path::preferred_separator};
101  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
102  base_path = foreign_server->getOption(BASE_PATH_KEY);
103  }
104 
105  // If both base_path and file_path are present, then concatenate. Otherwise we are just
106  // taking the one as the path. One of the two must exist, or we have failed validation.
107  CHECK(file_path || base_path);
108  return append_file_path(base_path, file_path);
109 }
std::string append_file_path(const std::optional< std::string > &base, const std::optional< std::string > &subdirectory)
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 52 of file AbstractFileStorageDataWrapper.cpp.

References supported_table_options_.

Referenced by foreign_storage::CsvDataWrapper::getAllCsvTableOptions(), and foreign_storage::RegexParserDataWrapper::getAllRegexTableOptions().

52  {
54 }
static const std::set< std::string_view > supported_table_options_

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedUserMappingOptions ( ) const
overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 65 of file AbstractFileStorageDataWrapper.cpp.

References supported_user_mapping_options_.

65  {
67 }
static const std::set< std::string_view > supported_user_mapping_options_
void foreign_storage::AbstractFileStorageDataWrapper::validateFilePath ( const ForeignTable foreign_table)
staticprivate

Definition at line 69 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, getFullFilePath(), ddl_utils::IMPORT, LOCAL_FILE_STORAGE_TYPE, foreign_storage::OptionsContainer::options, STORAGE_TYPE_KEY, and ddl_utils::validate_allowed_file_path().

Referenced by validateTableOptions().

69  {
70  auto& server_options = foreign_table->foreign_server->options;
71  if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
74  }
75 }
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:770
static std::string getFullFilePath(const ForeignTable *foreign_table)
Returns the path to the source file/dir of the table. Depending on options this may result from a con...

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePathOptionKey ( const ForeignTable foreign_table)
staticprivate

Definition at line 125 of file AbstractFileStorageDataWrapper.cpp.

References BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, STORAGE_TYPE_KEY, TableDescriptor::tableName, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::throw_file_path_error(), and UNREACHABLE.

Referenced by validateTableOptions().

126  {
127  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
128  auto foreign_server = foreign_table->foreign_server;
129  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
130  CHECK(storage_type) << "No storage type found in parent server. Server \""
131  << foreign_server->name << "\" is not valid.";
132  if (!file_path) {
133  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
134  if (!foreign_server->getOption(BASE_PATH_KEY)) {
135  throw_file_path_error(BASE_PATH_KEY, foreign_table->tableName, FILE_PATH_KEY);
136  }
137  } else {
138  UNREACHABLE() << "Unknown foreign storage type.";
139  }
140  }
141 }
void throw_file_path_error(const std::string_view &missing_path, const std::string &table_name, const std::string_view &file_path_key)
#define UNREACHABLE()
Definition: Logger.h:253
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateServerOptions ( const ForeignServer foreign_server) const
overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters
foreign_server- foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 14 of file AbstractFileStorageDataWrapper.cpp.

References shared::contains(), g_enable_s3_fsi, join(), foreign_storage::OptionsContainer::options, S3_STORAGE_TYPE, STORAGE_TYPE_KEY, supported_server_options_, and supported_storage_types.

15  {
16  const auto& options = foreign_server->options;
17  for (const auto& entry : options) {
18  if (!shared::contains(supported_server_options_, entry.first)) {
19  throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
20  "\". Option must be one of the following: " +
21  join(supported_server_options_, ", ") + "."};
22  }
23  }
24 
25  if (options.find(STORAGE_TYPE_KEY) == options.end()) {
26  throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
27  "\"."};
28  }
29  const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
30  if (!shared::contains(supported_storage_types, storage_type)) {
31  throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
32  "\" option value. Value must be one of the following: " +
33  join(supported_storage_types, ", ") + "."};
34  }
35 
36  if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
37  throw std::runtime_error{
38  "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
39  "\" is not allowed because FSI S3 support is currently disabled."};
40  }
41 }
bool contains(const T &container, const U &element)
Definition: misc.h:188
static const std::array< std::string, 1 > supported_storage_types
std::string join(T const &container, std::string const &delim)
static const std::set< std::string_view > supported_server_options_
bool g_enable_s3_fsi
Definition: Catalog.cpp:94

+ Here is the call graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions ( const ForeignTable foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 43 of file AbstractFileStorageDataWrapper.cpp.

References FILE_SORT_ORDER_BY_KEY, FILE_SORT_REGEX_KEY, foreign_storage::OptionsContainer::getOption(), shared::validate_sort_options(), validateFilePath(), and validateFilePathOptionKey().

Referenced by foreign_storage::CsvDataWrapper::validateTableOptions(), and foreign_storage::RegexParserDataWrapper::validateTableOptions().

44  {
45  validateFilePathOptionKey(foreign_table);
46  validateFilePath(foreign_table);
47  shared::validate_sort_options(foreign_table->getOption(FILE_SORT_ORDER_BY_KEY),
48  foreign_table->getOption(FILE_SORT_REGEX_KEY));
49 }
static void validateFilePathOptionKey(const ForeignTable *foreign_table)
void validate_sort_options(const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex)
static void validateFilePath(const ForeignTable *foreign_table)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateUserMappingOptions ( const UserMapping user_mapping,
const ForeignServer foreign_server 
) const
overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters
user_mapping- user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 56 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignServer::data_wrapper_type.

58  {
59  throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
60  "\" data wrapper can only be created for AWS S3 backed "
61  "foreign servers. AWS S3 support is currently disabled."};
62 }

Member Data Documentation

const std::string foreign_storage::AbstractFileStorageDataWrapper::BASE_PATH_KEY = "BASE_PATH"
inlinestatic

Definition at line 24 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_PATH_KEY = "FILE_PATH"
inlinestatic

Definition at line 25 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE = "AWS_S3"
inlinestatic
const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_server_options_
staticprivate
Initial value:

Definition at line 50 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::array<std::string, 1> foreign_storage::AbstractFileStorageDataWrapper::supported_storage_types
inlinestatic
Initial value:

Definition at line 32 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_table_options_
staticprivate
const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_user_mapping_options_ {}
staticprivate

Definition at line 51 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedUserMappingOptions().


The documentation for this class was generated from the following files: