OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::AbstractFileStorageDataWrapper Class Reference

#include <AbstractFileStorageDataWrapper.h>

+ Inheritance diagram for foreign_storage::AbstractFileStorageDataWrapper:
+ Collaboration diagram for foreign_storage::AbstractFileStorageDataWrapper:

Public Member Functions

 AbstractFileStorageDataWrapper ()
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 
getSupportedTableOptions () const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 
getSupportedUserMappingOptions () const override
 
const std::set< std::string > getAlterableTableOptions () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector)=0
 
virtual void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer=nullptr)=0
 
virtual std::string getSerializedDataWrapper () const =0
 
virtual void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata)=0
 
virtual bool isRestored () const =0
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 
virtual ParallelismLevel getCachedParallelismLevel () const
 
virtual ParallelismLevel getNonCachedParallelismLevel () const
 
virtual void createRenderGroupAnalyzers ()
 Create RenderGroupAnalyzers for poly columns. More...
 
virtual bool isLazyFragmentFetchingEnabled () const
 

Static Public Member Functions

static shared::FilePathOptions getFilePathOptions (const ForeignTable *foreign_table)
 

Static Public Attributes

static const std::string STORAGE_TYPE_KEY = "STORAGE_TYPE"
 
static const std::string BASE_PATH_KEY = "BASE_PATH"
 
static const std::string FILE_PATH_KEY = "FILE_PATH"
 
static const std::string REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
 
static const std::string LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"
 
static const std::string S3_STORAGE_TYPE = "AWS_S3"
 
static const std::string FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
 
static const std::string FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
 
static const std::string ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"
 
static const std::string THREADS_KEY = "THREADS"
 
static const std::array
< std::string, 1 > 
supported_storage_types
 

Static Protected Member Functions

static std::string getFullFilePath (const ForeignTable *foreign_table)
 Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...
 
static bool allowFileRollOff (const ForeignTable *foreign_table)
 

Static Private Member Functions

static void validateFilePath (const ForeignTable *foreign_table)
 
static void validateFilePathOptionKey (const ForeignTable *foreign_table)
 
static void validateFileRollOffOption (const ForeignTable *foreign_table)
 

Static Private Attributes

static const std::set
< std::string_view > 
supported_table_options_
 
static const std::set
< std::string_view > 
supported_server_options_
 
static const std::set
< std::string_view > 
supported_user_mapping_options_ {}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 

Detailed Description

Definition at line 11 of file AbstractFileStorageDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::AbstractFileStorageDataWrapper::AbstractFileStorageDataWrapper ( )

Definition at line 26 of file AbstractFileStorageDataWrapper.cpp.

26 {}

Member Function Documentation

bool foreign_storage::AbstractFileStorageDataWrapper::allowFileRollOff ( const ForeignTable foreign_table)
staticprotected

Definition at line 212 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, CHECK, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), foreign_storage::OptionsContainer::getOption(), and UNREACHABLE.

Referenced by foreign_storage::ParquetDataWrapper::addNewFile(), foreign_storage::ParquetDataWrapper::fetchChunkMetadata(), foreign_storage::ParquetDataWrapper::isNewFile(), and foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

212  {
213  auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
214  if (allow_file_roll_off.has_value()) {
215  return allow_file_roll_off.value();
216  } else {
217  auto option = foreign_table->getOption(ALLOW_FILE_ROLL_OFF_KEY);
218  CHECK(option.has_value());
219  UNREACHABLE() << "Unexpected " << ALLOW_FILE_ROLL_OFF_KEY
220  << " value: " << option.value();
221  return false;
222  }
223 }
#define UNREACHABLE()
Definition: Logger.h:337
std::optional< bool > get_file_roll_off_value(const ForeignTable *foreign_table)
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::set< std::string > foreign_storage::AbstractFileStorageDataWrapper::getAlterableTableOptions ( ) const
overridevirtual

Gets the subset of table options that can be altered for the data wrapper.

Reimplemented from foreign_storage::ForeignDataWrapper.

Definition at line 225 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY.

226  {
227  return {ALLOW_FILE_ROLL_OFF_KEY};
228 }
shared::FilePathOptions foreign_storage::AbstractFileStorageDataWrapper::getFilePathOptions ( const ForeignTable foreign_table)
static

Definition at line 171 of file AbstractFileStorageDataWrapper.cpp.

References FILE_SORT_ORDER_BY_KEY, FILE_SORT_REGEX_KEY, foreign_storage::OptionsContainer::getOption(), and REGEX_PATH_FILTER_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and validateTableOptions().

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string foreign_storage::AbstractFileStorageDataWrapper::getFullFilePath ( const ForeignTable foreign_table)
staticprotected

Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options.

Definition at line 113 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::append_file_path(), BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, generate_TableFunctionsFactory_init::separator, and STORAGE_TYPE_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::ParquetImporter::getAllFilePaths(), foreign_storage::ParquetDataWrapper::getDataPreview(), foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), foreign_storage::AbstractTextFileDataWrapper::populateChunks(), foreign_storage::AbstractTextFileDataWrapper::restoreDataWrapperInternals(), and validateFilePath().

114  {
115  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
116  std::optional<std::string> base_path{};
117  auto foreign_server = foreign_table->foreign_server;
118  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
119  CHECK(storage_type);
120 
121 #ifdef _WIN32
122  const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
123  using convert_type = std::codecvt_utf8<wchar_t>;
124  std::wstring_convert<convert_type, wchar_t> converter;
125  std::string separator = converter.to_bytes(str_to_cov);
126 #else
127  const std::string separator{boost::filesystem::path::preferred_separator};
128 #endif
129  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
130  base_path = foreign_server->getOption(BASE_PATH_KEY);
131  }
132 
133  // If both base_path and file_path are present, then concatenate. Otherwise we are just
134  // taking the one as the path. One of the two must exist, or we have failed validation.
135  CHECK(file_path || base_path);
136  return append_file_path(base_path, file_path);
137 }
std::string append_file_path(const std::optional< std::string > &base, const std::optional< std::string > &subdirectory)
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 66 of file AbstractFileStorageDataWrapper.cpp.

References supported_table_options_.

Referenced by foreign_storage::CsvDataWrapper::getAllCsvTableOptions(), and foreign_storage::RegexParserDataWrapper::getAllRegexTableOptions().

66  {
68 }
static const std::set< std::string_view > supported_table_options_

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedUserMappingOptions ( ) const
overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 79 of file AbstractFileStorageDataWrapper.cpp.

References supported_user_mapping_options_.

79  {
81 }
static const std::set< std::string_view > supported_user_mapping_options_
void foreign_storage::AbstractFileStorageDataWrapper::validateFilePath ( const ForeignTable foreign_table)
staticprivate

Definition at line 83 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, getFullFilePath(), ddl_utils::IMPORT, LOCAL_FILE_STORAGE_TYPE, foreign_storage::OptionsContainer::options, STORAGE_TYPE_KEY, and ddl_utils::validate_allowed_file_path().

Referenced by validateTableOptions().

83  {
84  auto& server_options = foreign_table->foreign_server->options;
85  if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
88  }
89 }
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:785
static std::string getFullFilePath(const ForeignTable *foreign_table)
Returns the path to the source file/dir of the table. Depending on options this may result from a con...

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePathOptionKey ( const ForeignTable foreign_table)
staticprivate

Definition at line 153 of file AbstractFileStorageDataWrapper.cpp.

References BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, STORAGE_TYPE_KEY, TableDescriptor::tableName, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::throw_file_path_error(), and UNREACHABLE.

Referenced by validateTableOptions().

154  {
155  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
156  auto foreign_server = foreign_table->foreign_server;
157  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
158  CHECK(storage_type) << "No storage type found in parent server. Server \""
159  << foreign_server->name << "\" is not valid.";
160  if (!file_path) {
161  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
162  if (!foreign_server->getOption(BASE_PATH_KEY)) {
163  throw_file_path_error(BASE_PATH_KEY, foreign_table->tableName, FILE_PATH_KEY);
164  }
165  } else {
166  UNREACHABLE() << "Unknown foreign storage type.";
167  }
168  }
169 }
void throw_file_path_error(const std::string_view &missing_path, const std::string &table_name, const std::string_view &file_path_key)
#define UNREACHABLE()
Definition: Logger.h:337
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFileRollOffOption ( const ForeignTable foreign_table)
staticprivate

Definition at line 196 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), and foreign_storage::ForeignTable::isAppendMode().

Referenced by validateTableOptions().

197  {
198  auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
199  if (allow_file_roll_off.has_value()) {
200  if (allow_file_roll_off.value() && !foreign_table->isAppendMode()) {
201  throw std::runtime_error{"The \"" + ALLOW_FILE_ROLL_OFF_KEY +
202  "\" option can only be set to 'true' for foreign tables "
203  "with append refresh updates."};
204  }
205  } else {
206  throw std::runtime_error{
207  "Invalid boolean value specified for \"" + ALLOW_FILE_ROLL_OFF_KEY +
208  "\" foreign table option. Value must be either 'true' or 'false'."};
209  }
210 }
std::optional< bool > get_file_roll_off_value(const ForeignTable *foreign_table)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateServerOptions ( const ForeignServer foreign_server) const
overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters
foreign_server- foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 28 of file AbstractFileStorageDataWrapper.cpp.

References shared::contains(), g_enable_s3_fsi, join(), foreign_storage::OptionsContainer::options, S3_STORAGE_TYPE, STORAGE_TYPE_KEY, supported_server_options_, and supported_storage_types.

29  {
30  const auto& options = foreign_server->options;
31  for (const auto& entry : options) {
32  if (!shared::contains(supported_server_options_, entry.first)) {
33  throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
34  "\". Option must be one of the following: " +
35  join(supported_server_options_, ", ") + "."};
36  }
37  }
38 
39  if (options.find(STORAGE_TYPE_KEY) == options.end()) {
40  throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
41  "\"."};
42  }
43  const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
44  if (!shared::contains(supported_storage_types, storage_type)) {
45  throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
46  "\" option value. Value must be one of the following: " +
47  join(supported_storage_types, ", ") + "."};
48  }
49 
50  if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
51  throw std::runtime_error{
52  "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
53  "\" is not allowed because FSI S3 support is currently disabled."};
54  }
55 }
bool contains(const T &container, const U &element)
Definition: misc.h:195
static const std::array< std::string, 1 > supported_storage_types
std::string join(T const &container, std::string const &delim)
static const std::set< std::string_view > supported_server_options_
bool g_enable_s3_fsi
Definition: Catalog.cpp:97

+ Here is the call graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions ( const ForeignTable foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 57 of file AbstractFileStorageDataWrapper.cpp.

References getFilePathOptions(), shared::validate_sort_options(), validateFilePath(), validateFilePathOptionKey(), and validateFileRollOffOption().

Referenced by foreign_storage::CsvDataWrapper::validateTableOptions(), and foreign_storage::RegexParserDataWrapper::validateTableOptions().

58  {
59  validateFilePathOptionKey(foreign_table);
60  validateFilePath(foreign_table);
62  validateFileRollOffOption(foreign_table);
63 }
static void validateFilePathOptionKey(const ForeignTable *foreign_table)
static shared::FilePathOptions getFilePathOptions(const ForeignTable *foreign_table)
void validate_sort_options(const FilePathOptions &options)
static void validateFileRollOffOption(const ForeignTable *foreign_table)
static void validateFilePath(const ForeignTable *foreign_table)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateUserMappingOptions ( const UserMapping user_mapping,
const ForeignServer foreign_server 
) const
overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters
user_mapping- user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 70 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignServer::data_wrapper_type.

72  {
73  throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
74  "\" data wrapper can only be created for AWS S3 backed "
75  "foreign servers. AWS S3 support is currently disabled."};
76 }

Member Data Documentation

const std::string foreign_storage::AbstractFileStorageDataWrapper::ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::BASE_PATH_KEY = "BASE_PATH"
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_PATH_KEY = "FILE_PATH"
inlinestatic

Definition at line 32 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
inlinestatic
const std::string foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE = "AWS_S3"
inlinestatic
const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_server_options_
staticprivate
Initial value:

Definition at line 63 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::array<std::string, 1> foreign_storage::AbstractFileStorageDataWrapper::supported_storage_types
inlinestatic
Initial value:

Definition at line 41 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_table_options_
staticprivate
const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_user_mapping_options_ {}
staticprivate

Definition at line 64 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedUserMappingOptions().


The documentation for this class was generated from the following files: