OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::AbstractFileStorageDataWrapper Class Reference

#include <AbstractFileStorageDataWrapper.h>

+ Inheritance diagram for foreign_storage::AbstractFileStorageDataWrapper:
+ Collaboration diagram for foreign_storage::AbstractFileStorageDataWrapper:

Public Member Functions

 AbstractFileStorageDataWrapper ()
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 
getSupportedTableOptions () const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 
getSupportedUserMappingOptions () const override
 
const std::set< std::string > getAlterableTableOptions () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector)=0
 
virtual void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer=nullptr)=0
 
virtual std::string getSerializedDataWrapper () const =0
 
virtual void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata)=0
 
virtual bool isRestored () const =0
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 
virtual ParallelismLevel getCachedParallelismLevel () const
 
virtual ParallelismLevel getNonCachedParallelismLevel () const
 
virtual void createRenderGroupAnalyzers ()
 Create RenderGroupAnalyzers for poly columns. More...
 

Static Public Member Functions

static shared::FilePathOptions getFilePathOptions (const ForeignTable *foreign_table)
 

Static Public Attributes

static const std::string STORAGE_TYPE_KEY = "STORAGE_TYPE"
 
static const std::string BASE_PATH_KEY = "BASE_PATH"
 
static const std::string FILE_PATH_KEY = "FILE_PATH"
 
static const std::string REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
 
static const std::string LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"
 
static const std::string S3_STORAGE_TYPE = "AWS_S3"
 
static const std::string FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
 
static const std::string FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
 
static const std::string ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"
 
static const std::array
< std::string, 1 > 
supported_storage_types
 

Static Protected Member Functions

static std::string getFullFilePath (const ForeignTable *foreign_table)
 Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...
 
static bool allowFileRollOff (const ForeignTable *foreign_table)
 

Static Private Member Functions

static void validateFilePath (const ForeignTable *foreign_table)
 
static void validateFilePathOptionKey (const ForeignTable *foreign_table)
 
static void validateFileRollOffOption (const ForeignTable *foreign_table)
 

Static Private Attributes

static const std::set
< std::string_view > 
supported_table_options_
 
static const std::set
< std::string_view > 
supported_server_options_
 
static const std::set
< std::string_view > 
supported_user_mapping_options_ {}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 

Detailed Description

Definition at line 8 of file AbstractFileStorageDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::AbstractFileStorageDataWrapper::AbstractFileStorageDataWrapper ( )

Definition at line 15 of file AbstractFileStorageDataWrapper.cpp.

15 {}

Member Function Documentation

bool foreign_storage::AbstractFileStorageDataWrapper::allowFileRollOff ( const ForeignTable foreign_table)
staticprotected

Definition at line 201 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, CHECK, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), foreign_storage::OptionsContainer::getOption(), and UNREACHABLE.

Referenced by foreign_storage::ParquetDataWrapper::addNewFile(), foreign_storage::ParquetDataWrapper::fetchChunkMetadata(), foreign_storage::ParquetDataWrapper::isNewFile(), and foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

201  {
202  auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
203  if (allow_file_roll_off.has_value()) {
204  return allow_file_roll_off.value();
205  } else {
206  auto option = foreign_table->getOption(ALLOW_FILE_ROLL_OFF_KEY);
207  CHECK(option.has_value());
208  UNREACHABLE() << "Unexpected " << ALLOW_FILE_ROLL_OFF_KEY
209  << " value: " << option.value();
210  return false;
211  }
212 }
#define UNREACHABLE()
Definition: Logger.h:266
std::optional< bool > get_file_roll_off_value(const ForeignTable *foreign_table)
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::set< std::string > foreign_storage::AbstractFileStorageDataWrapper::getAlterableTableOptions ( ) const
overridevirtual

Gets the subset of table options that can be altered for the data wrapper.

Reimplemented from foreign_storage::ForeignDataWrapper.

Definition at line 214 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY.

215  {
216  return {ALLOW_FILE_ROLL_OFF_KEY};
217 }
shared::FilePathOptions foreign_storage::AbstractFileStorageDataWrapper::getFilePathOptions ( const ForeignTable foreign_table)
static

Definition at line 160 of file AbstractFileStorageDataWrapper.cpp.

References FILE_SORT_ORDER_BY_KEY, FILE_SORT_REGEX_KEY, foreign_storage::OptionsContainer::getOption(), and REGEX_PATH_FILTER_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and validateTableOptions().

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string foreign_storage::AbstractFileStorageDataWrapper::getFullFilePath ( const ForeignTable foreign_table)
staticprotected

Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options.

Definition at line 102 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::append_file_path(), BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, generate_TableFunctionsFactory_init::separator, and STORAGE_TYPE_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::ParquetImporter::getAllFilePaths(), foreign_storage::ParquetDataWrapper::getDataPreview(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), foreign_storage::AbstractTextFileDataWrapper::populateChunks(), foreign_storage::AbstractTextFileDataWrapper::restoreDataWrapperInternals(), and validateFilePath().

103  {
104  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
105  std::optional<std::string> base_path{};
106  auto foreign_server = foreign_table->foreign_server;
107  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
108  CHECK(storage_type);
109 
110 #ifdef _WIN32
111  const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
112  using convert_type = std::codecvt_utf8<wchar_t>;
113  std::wstring_convert<convert_type, wchar_t> converter;
114  std::string separator = converter.to_bytes(str_to_cov);
115 #else
116  const std::string separator{boost::filesystem::path::preferred_separator};
117 #endif
118  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
119  base_path = foreign_server->getOption(BASE_PATH_KEY);
120  }
121 
122  // If both base_path and file_path are present, then concatenate. Otherwise we are just
123  // taking the one as the path. One of the two must exist, or we have failed validation.
124  CHECK(file_path || base_path);
125  return append_file_path(base_path, file_path);
126 }
std::string append_file_path(const std::optional< std::string > &base, const std::optional< std::string > &subdirectory)
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 55 of file AbstractFileStorageDataWrapper.cpp.

References supported_table_options_.

Referenced by foreign_storage::CsvDataWrapper::getAllCsvTableOptions(), and foreign_storage::RegexParserDataWrapper::getAllRegexTableOptions().

55  {
57 }
static const std::set< std::string_view > supported_table_options_

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedUserMappingOptions ( ) const
overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 68 of file AbstractFileStorageDataWrapper.cpp.

References supported_user_mapping_options_.

68  {
70 }
static const std::set< std::string_view > supported_user_mapping_options_
void foreign_storage::AbstractFileStorageDataWrapper::validateFilePath ( const ForeignTable foreign_table)
staticprivate

Definition at line 72 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, getFullFilePath(), ddl_utils::IMPORT, LOCAL_FILE_STORAGE_TYPE, foreign_storage::OptionsContainer::options, STORAGE_TYPE_KEY, and ddl_utils::validate_allowed_file_path().

Referenced by validateTableOptions().

72  {
73  auto& server_options = foreign_table->foreign_server->options;
74  if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
77  }
78 }
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:770
static std::string getFullFilePath(const ForeignTable *foreign_table)
Returns the path to the source file/dir of the table. Depending on options this may result from a con...

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePathOptionKey ( const ForeignTable foreign_table)
staticprivate

Definition at line 142 of file AbstractFileStorageDataWrapper.cpp.

References BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, STORAGE_TYPE_KEY, TableDescriptor::tableName, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::throw_file_path_error(), and UNREACHABLE.

Referenced by validateTableOptions().

143  {
144  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
145  auto foreign_server = foreign_table->foreign_server;
146  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
147  CHECK(storage_type) << "No storage type found in parent server. Server \""
148  << foreign_server->name << "\" is not valid.";
149  if (!file_path) {
150  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
151  if (!foreign_server->getOption(BASE_PATH_KEY)) {
152  throw_file_path_error(BASE_PATH_KEY, foreign_table->tableName, FILE_PATH_KEY);
153  }
154  } else {
155  UNREACHABLE() << "Unknown foreign storage type.";
156  }
157  }
158 }
void throw_file_path_error(const std::string_view &missing_path, const std::string &table_name, const std::string_view &file_path_key)
#define UNREACHABLE()
Definition: Logger.h:266
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFileRollOffOption ( const ForeignTable foreign_table)
staticprivate

Definition at line 185 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), and foreign_storage::ForeignTable::isAppendMode().

Referenced by validateTableOptions().

186  {
187  auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
188  if (allow_file_roll_off.has_value()) {
189  if (allow_file_roll_off.value() && !foreign_table->isAppendMode()) {
190  throw std::runtime_error{"The \"" + ALLOW_FILE_ROLL_OFF_KEY +
191  "\" option can only be set to 'true' for foreign tables "
192  "with append refresh updates."};
193  }
194  } else {
195  throw std::runtime_error{
196  "Invalid boolean value specified for \"" + ALLOW_FILE_ROLL_OFF_KEY +
197  "\" foreign table option. Value must be either 'true' or 'false'."};
198  }
199 }
std::optional< bool > get_file_roll_off_value(const ForeignTable *foreign_table)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateServerOptions ( const ForeignServer foreign_server) const
overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters
foreign_server- foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 17 of file AbstractFileStorageDataWrapper.cpp.

References shared::contains(), g_enable_s3_fsi, join(), foreign_storage::OptionsContainer::options, S3_STORAGE_TYPE, STORAGE_TYPE_KEY, supported_server_options_, and supported_storage_types.

18  {
19  const auto& options = foreign_server->options;
20  for (const auto& entry : options) {
21  if (!shared::contains(supported_server_options_, entry.first)) {
22  throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
23  "\". Option must be one of the following: " +
24  join(supported_server_options_, ", ") + "."};
25  }
26  }
27 
28  if (options.find(STORAGE_TYPE_KEY) == options.end()) {
29  throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
30  "\"."};
31  }
32  const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
33  if (!shared::contains(supported_storage_types, storage_type)) {
34  throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
35  "\" option value. Value must be one of the following: " +
36  join(supported_storage_types, ", ") + "."};
37  }
38 
39  if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
40  throw std::runtime_error{
41  "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
42  "\" is not allowed because FSI S3 support is currently disabled."};
43  }
44 }
bool contains(const T &container, const U &element)
Definition: misc.h:195
static const std::array< std::string, 1 > supported_storage_types
std::string join(T const &container, std::string const &delim)
static const std::set< std::string_view > supported_server_options_
bool g_enable_s3_fsi
Definition: Catalog.cpp:97

+ Here is the call graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions ( const ForeignTable foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 46 of file AbstractFileStorageDataWrapper.cpp.

References getFilePathOptions(), shared::validate_sort_options(), validateFilePath(), validateFilePathOptionKey(), and validateFileRollOffOption().

Referenced by foreign_storage::CsvDataWrapper::validateTableOptions(), and foreign_storage::RegexParserDataWrapper::validateTableOptions().

47  {
48  validateFilePathOptionKey(foreign_table);
49  validateFilePath(foreign_table);
51  validateFileRollOffOption(foreign_table);
52 }
static void validateFilePathOptionKey(const ForeignTable *foreign_table)
static shared::FilePathOptions getFilePathOptions(const ForeignTable *foreign_table)
void validate_sort_options(const FilePathOptions &options)
static void validateFileRollOffOption(const ForeignTable *foreign_table)
static void validateFilePath(const ForeignTable *foreign_table)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateUserMappingOptions ( const UserMapping user_mapping,
const ForeignServer foreign_server 
) const
overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters
user_mapping- user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 59 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignServer::data_wrapper_type.

61  {
62  throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
63  "\" data wrapper can only be created for AWS S3 backed "
64  "foreign servers. AWS S3 support is currently disabled."};
65 }

Member Data Documentation

const std::string foreign_storage::AbstractFileStorageDataWrapper::ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::BASE_PATH_KEY = "BASE_PATH"
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_PATH_KEY = "FILE_PATH"
inlinestatic

Definition at line 29 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE = "AWS_S3"
inlinestatic
const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_server_options_
staticprivate
Initial value:

Definition at line 59 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::array<std::string, 1> foreign_storage::AbstractFileStorageDataWrapper::supported_storage_types
inlinestatic
Initial value:

Definition at line 37 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_table_options_
staticprivate
const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_user_mapping_options_ {}
staticprivate

Definition at line 60 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedUserMappingOptions().


The documentation for this class was generated from the following files: