OmniSciDB  a667adc9c8
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::AbstractFileStorageDataWrapper Class Reference

#include <AbstractFileStorageDataWrapper.h>

+ Inheritance diagram for foreign_storage::AbstractFileStorageDataWrapper:
+ Collaboration diagram for foreign_storage::AbstractFileStorageDataWrapper:

Public Member Functions

 AbstractFileStorageDataWrapper ()
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 
getSupportedTableOptions () const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 
getSupportedUserMappingOptions () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector)=0
 
virtual void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers)=0
 
virtual void serializeDataWrapperInternals (const std::string &file_path) const =0
 
virtual void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata)=0
 
virtual bool isRestored () const =0
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 
virtual ParallelismLevel getCachedParallelismLevel () const
 
virtual ParallelismLevel getNonCachedParallelismLevel () const
 

Static Public Attributes

static const std::string STORAGE_TYPE_KEY = "STORAGE_TYPE"
 
static const std::string BASE_PATH_KEY = "BASE_PATH"
 
static const std::string FILE_PATH_KEY = "FILE_PATH"
 
static const std::string LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"
 
static const std::string S3_STORAGE_TYPE = "AWS_S3"
 
static const std::array
< std::string, 1 > 
supported_storage_types
 

Static Protected Member Functions

static std::string getFullFilePath (const ForeignTable *foreign_table)
 Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...
 

Static Private Member Functions

static void validateFilePath (const ForeignTable *foreign_table)
 
static void validateFilePathOptionKey (const ForeignTable *foreign_table)
 

Static Private Attributes

static const std::set
< std::string_view > 
supported_table_options_
 
static const std::set
< std::string_view > 
supported_server_options_
 
static const std::set
< std::string_view > 
supported_user_mapping_options_ {}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 

Detailed Description

Definition at line 6 of file AbstractFileStorageDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::AbstractFileStorageDataWrapper::AbstractFileStorageDataWrapper ( )

Definition at line 11 of file AbstractFileStorageDataWrapper.cpp.

11 {}

Member Function Documentation

std::string foreign_storage::AbstractFileStorageDataWrapper::getFullFilePath ( const ForeignTable foreign_table)
staticprotected

Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options.

Definition at line 89 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::append_file_path(), BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, and STORAGE_TYPE_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::CsvDataWrapper::populateChunkMetadata(), foreign_storage::CsvDataWrapper::populateChunks(), foreign_storage::CsvDataWrapper::restoreDataWrapperInternals(), and validateFilePath().

90  {
91  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
92  std::optional<std::string> base_path{};
93  auto foreign_server = foreign_table->foreign_server;
94  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
95  CHECK(storage_type);
96 
97  const std::string separator{boost::filesystem::path::preferred_separator};
98  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
99  base_path = foreign_server->getOption(BASE_PATH_KEY);
100  }
101 
102  // If both base_path and file_path are present, then concatenate. Otherwise we are just
103  // taking the one as the path. One of the two must exist, or we have failed validation.
104  CHECK(file_path || base_path);
105  return append_file_path(base_path, file_path);
106 }
std::string append_file_path(const std::optional< std::string > &base, const std::optional< std::string > &subdirectory)
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper.

Definition at line 49 of file AbstractFileStorageDataWrapper.cpp.

References supported_table_options_.

Referenced by foreign_storage::CsvDataWrapper::getAllCsvTableOptions().

49  {
51 }
static const std::set< std::string_view > supported_table_options_

+ Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedUserMappingOptions ( ) const
overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 62 of file AbstractFileStorageDataWrapper.cpp.

References supported_user_mapping_options_.

62  {
64 }
static const std::set< std::string_view > supported_user_mapping_options_
void foreign_storage::AbstractFileStorageDataWrapper::validateFilePath ( const ForeignTable foreign_table)
staticprivate

Definition at line 66 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, getFullFilePath(), ddl_utils::IMPORT, LOCAL_FILE_STORAGE_TYPE, foreign_storage::OptionsContainer::options, STORAGE_TYPE_KEY, and ddl_utils::validate_allowed_file_path().

Referenced by validateTableOptions().

66  {
67  auto& server_options = foreign_table->foreign_server->options;
68  if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
71  }
72 }
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:613
static std::string getFullFilePath(const ForeignTable *foreign_table)
Returns the path to the source file/dir of the table. Depending on options this may result from a con...

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePathOptionKey ( const ForeignTable foreign_table)
staticprivate

Definition at line 122 of file AbstractFileStorageDataWrapper.cpp.

References BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, STORAGE_TYPE_KEY, TableDescriptor::tableName, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::throw_file_path_error(), and UNREACHABLE.

Referenced by validateTableOptions().

123  {
124  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
125  auto foreign_server = foreign_table->foreign_server;
126  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
127  CHECK(storage_type) << "No storage type found in parent server. Server \""
128  << foreign_server->name << "\" is not valid.";
129  if (!file_path) {
130  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
131  if (!foreign_server->getOption(BASE_PATH_KEY)) {
132  throw_file_path_error(BASE_PATH_KEY, foreign_table->tableName, FILE_PATH_KEY);
133  }
134  } else {
135  UNREACHABLE() << "Unknown foreign storage type.";
136  }
137  }
138 }
void throw_file_path_error(const std::string_view &missing_path, const std::string &table_name, const std::string_view &file_path_key)
#define UNREACHABLE()
Definition: Logger.h:241
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateServerOptions ( const ForeignServer foreign_server) const
overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters
foreign_server- foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 13 of file AbstractFileStorageDataWrapper.cpp.

References shared::contains(), g_enable_s3_fsi, join(), foreign_storage::OptionsContainer::options, S3_STORAGE_TYPE, STORAGE_TYPE_KEY, supported_server_options_, and supported_storage_types.

14  {
15  const auto& options = foreign_server->options;
16  for (const auto& entry : options) {
17  if (!shared::contains(supported_server_options_, entry.first)) {
18  throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
19  "\". Option must be one of the following: " +
20  join(supported_server_options_, ", ") + "."};
21  }
22  }
23 
24  if (options.find(STORAGE_TYPE_KEY) == options.end()) {
25  throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
26  "\"."};
27  }
28  const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
29  if (!shared::contains(supported_storage_types, storage_type)) {
30  throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
31  "\" option value. Value must be one of the following: " +
32  join(supported_storage_types, ", ") + "."};
33  }
34 
35  if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
36  throw std::runtime_error{
37  "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
38  "\" is not allowed because FSI S3 support is currently disabled."};
39  }
40 }
bool contains(const T &container, const U &element)
Definition: misc.h:147
static const std::array< std::string, 1 > supported_storage_types
std::string join(T const &container, std::string const &delim)
static const std::set< std::string_view > supported_server_options_
bool g_enable_s3_fsi
Definition: Catalog.cpp:93

+ Here is the call graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions ( const ForeignTable foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper.

Definition at line 42 of file AbstractFileStorageDataWrapper.cpp.

References validateFilePath(), and validateFilePathOptionKey().

Referenced by foreign_storage::CsvDataWrapper::validateTableOptions().

43  {
44  validateFilePathOptionKey(foreign_table);
45  validateFilePath(foreign_table);
46 }
static void validateFilePathOptionKey(const ForeignTable *foreign_table)
static void validateFilePath(const ForeignTable *foreign_table)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateUserMappingOptions ( const UserMapping *  user_mapping,
const ForeignServer foreign_server 
) const
overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters
user_mapping- user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 53 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignServer::data_wrapper_type.

55  {
56  throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
57  "\" data wrapper can only be created for AWS S3 backed "
58  "foreign servers. AWS S3 support is currently disabled."};
59 }

Member Data Documentation

const std::string foreign_storage::AbstractFileStorageDataWrapper::BASE_PATH_KEY = "BASE_PATH"
inlinestatic

Definition at line 22 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_PATH_KEY = "FILE_PATH"
inlinestatic

Definition at line 23 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE = "AWS_S3"
inlinestatic
const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_server_options_
staticprivate
Initial value:

Definition at line 45 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::array<std::string, 1> foreign_storage::AbstractFileStorageDataWrapper::supported_storage_types
inlinestatic
Initial value:

Definition at line 27 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_table_options_
staticprivate
Initial value:

Definition at line 44 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedTableOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_user_mapping_options_ {}
staticprivate

Definition at line 46 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedUserMappingOptions().


The documentation for this class was generated from the following files: