#include <AbstractFileStorageDataWrapper.h>

Inheritance diagram for foreign_storage::AbstractFileStorageDataWrapper:

Collaboration diagram for foreign_storage::AbstractFileStorageDataWrapper:

Public Member Functions
	AbstractFileStorageDataWrapper ()

void	validateServerOptions (const ForeignServer *foreign_server) const override

void	validateTableOptions (const ForeignTable *foreign_table) const override

const std::set < std::string_view > &	getSupportedTableOptions () const override

void	validateUserMappingOptions (const UserMapping user_mapping, const ForeignServer foreign_server) const override

const std::set < std::string_view > &	getSupportedUserMappingOptions () const override

const std::set< std::string >	getAlterableTableOptions () const override

Public Member Functions inherited from foreign_storage::ForeignDataWrapper
	ForeignDataWrapper ()=default

virtual	~ForeignDataWrapper ()=default

virtual void	populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector)=0

virtual void	populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer=nullptr)=0

virtual std::string	getSerializedDataWrapper () const =0

virtual void	restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata)=0

virtual bool	isRestored () const =0

virtual void	validateSchema (const std::list< ColumnDescriptor > &columns) const

virtual ParallelismLevel	getCachedParallelismLevel () const

virtual ParallelismLevel	getNonCachedParallelismLevel () const

virtual bool	isLazyFragmentFetchingEnabled () const

Static Public Member Functions
static shared::FilePathOptions	getFilePathOptions (const ForeignTable *foreign_table)

Static Public Attributes
static const std::string	STORAGE_TYPE_KEY = "STORAGE_TYPE"

static const std::string	BASE_PATH_KEY = "BASE_PATH"

static const std::string	FILE_PATH_KEY = "FILE_PATH"

static const std::string	REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"

static const std::string	LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"

static const std::string	S3_STORAGE_TYPE = "AWS_S3"

static const std::string	FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY

static const std::string	FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY

static const std::string	ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"

static const std::string	THREADS_KEY = "THREADS"

static const std::array < std::string, 1 >	supported_storage_types

Static Protected Member Functions
static std::string	getFullFilePath (const ForeignTable *foreign_table)
	Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...

static bool	allowFileRollOff (const ForeignTable *foreign_table)

Static Private Member Functions
static void	validateFilePath (const ForeignTable *foreign_table)

static void	validateFilePathOptionKey (const ForeignTable *foreign_table)

static void	validateFileRollOffOption (const ForeignTable *foreign_table)

Static Private Attributes
static const std::set < std::string_view >	supported_table_options_

static const std::set < std::string_view >	supported_server_options_

static const std::set < std::string_view >	supported_user_mapping_options_ {}

Additional Inherited Members
Public Types inherited from foreign_storage::ForeignDataWrapper
enum	ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }

Detailed Description

Definition at line 11 of file AbstractFileStorageDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::AbstractFileStorageDataWrapper::AbstractFileStorageDataWrapper ( )

Definition at line 26 of file AbstractFileStorageDataWrapper.cpp.

26 {}

Member Function Documentation

bool foreign_storage::AbstractFileStorageDataWrapper::allowFileRollOff ( const ForeignTable * foreign_table )

staticprotected

Definition at line 212 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, CHECK, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), foreign_storage::OptionsContainer::getOption(), and UNREACHABLE.

Referenced by foreign_storage::ParquetDataWrapper::addNewFile(), foreign_storage::ParquetDataWrapper::fetchChunkMetadata(), foreign_storage::ParquetDataWrapper::isNewFile(), and foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                                                                        {
   auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
   if (allow_file_roll_off.has_value()) {
     return allow_file_roll_off.value();
   } else {
     auto option = foreign_table->getOption(ALLOW_FILE_ROLL_OFF_KEY);
     CHECK(option.has_value());
     UNREACHABLE() << "Unexpected " << ALLOW_FILE_ROLL_OFF_KEY
                   << " value: " << option.value();
     return false;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const std::set< std::string > foreign_storage::AbstractFileStorageDataWrapper::getAlterableTableOptions ( ) const

overridevirtual

Gets the subset of table options that can be altered for the data wrapper.

Reimplemented from foreign_storage::ForeignDataWrapper.

Definition at line 225 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY.

           {
   return {ALLOW_FILE_ROLL_OFF_KEY};
 }

shared::FilePathOptions foreign_storage::AbstractFileStorageDataWrapper::getFilePathOptions ( const ForeignTable * foreign_table )

static

Definition at line 171 of file AbstractFileStorageDataWrapper.cpp.

References FILE_SORT_ORDER_BY_KEY, FILE_SORT_REGEX_KEY, foreign_storage::OptionsContainer::getOption(), and REGEX_PATH_FILTER_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and validateTableOptions().

                                        {
   return {
       foreign_table->getOption(AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY),
       foreign_table->getOption(AbstractFileStorageDataWrapper::FILE_SORT_ORDER_BY_KEY),
       foreign_table->getOption(AbstractFileStorageDataWrapper::FILE_SORT_REGEX_KEY)};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::AbstractFileStorageDataWrapper::getFullFilePath ( const ForeignTable * foreign_table )

staticprotected

Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options.

Definition at line 113 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::append_file_path(), BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, generate_TableFunctionsFactory_init::separator, and STORAGE_TYPE_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::ParquetImporter::getAllFilePaths(), foreign_storage::ParquetDataWrapper::getDataPreview(), foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), foreign_storage::AbstractTextFileDataWrapper::populateChunks(), foreign_storage::AbstractTextFileDataWrapper::restoreDataWrapperInternals(), and validateFilePath().

                                        {
   auto file_path = foreign_table->getOption(FILE_PATH_KEY);
   std::optional<std::string> base_path{};
   auto foreign_server = foreign_table->foreign_server;
   auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
   CHECK(storage_type);
 
 #ifdef _WIN32
   const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
   using convert_type = std::codecvt_utf8<wchar_t>;
   std::wstring_convert<convert_type, wchar_t> converter;
   std::string separator = converter.to_bytes(str_to_cov);
 #else
   const std::string separator{boost::filesystem::path::preferred_separator};
 #endif
   if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
     base_path = foreign_server->getOption(BASE_PATH_KEY);
   }
 
   // If both base_path and file_path are present, then concatenate.  Otherwise we are just
   // taking the one as the path.  One of the two must exist, or we have failed validation.
   CHECK(file_path || base_path);
   return append_file_path(base_path, file_path);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions ( ) const

overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 66 of file AbstractFileStorageDataWrapper.cpp.

References supported_table_options_.

Referenced by foreign_storage::CsvDataWrapper::getAllCsvTableOptions(), and foreign_storage::RegexParserDataWrapper::getAllRegexTableOptions().

                                                                {
   return supported_table_options_;
 }

Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedUserMappingOptions ( ) const

overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 79 of file AbstractFileStorageDataWrapper.cpp.

References supported_user_mapping_options_.

                                                                      {
   return supported_user_mapping_options_;
 }

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePath ( const ForeignTable * foreign_table )

staticprivate

Definition at line 83 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, getFullFilePath(), ddl_utils::IMPORT, LOCAL_FILE_STORAGE_TYPE, foreign_storage::OptionsContainer::options, STORAGE_TYPE_KEY, and ddl_utils::validate_allowed_file_path().

Referenced by validateTableOptions().

                                                                                        {
   auto& server_options = foreign_table->foreign_server->options;
   if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
     ddl_utils::validate_allowed_file_path(
         getFullFilePath(foreign_table), ddl_utils::DataTransferType::IMPORT, true);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePathOptionKey ( const ForeignTable * foreign_table )

staticprivate

Definition at line 153 of file AbstractFileStorageDataWrapper.cpp.

References BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, STORAGE_TYPE_KEY, TableDescriptor::tableName, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::throw_file_path_error(), and UNREACHABLE.

Referenced by validateTableOptions().

                                        {
   auto file_path = foreign_table->getOption(FILE_PATH_KEY);
   auto foreign_server = foreign_table->foreign_server;
   auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
   CHECK(storage_type) << "No storage type found in parent server. Server \""
                       << foreign_server->name << "\" is not valid.";
   if (!file_path) {
     if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
       if (!foreign_server->getOption(BASE_PATH_KEY)) {
         throw_file_path_error(BASE_PATH_KEY, foreign_table->tableName, FILE_PATH_KEY);
       }
     } else {
       UNREACHABLE() << "Unknown foreign storage type.";
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFileRollOffOption ( const ForeignTable * foreign_table )

staticprivate

Definition at line 196 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), and foreign_storage::ForeignTable::isAppendMode().

Referenced by validateTableOptions().

                                        {
   auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
   if (allow_file_roll_off.has_value()) {
     if (allow_file_roll_off.value() && !foreign_table->isAppendMode()) {
       throw std::runtime_error{"The \"" + ALLOW_FILE_ROLL_OFF_KEY +
                                "\" option can only be set to 'true' for foreign tables "
                                "with append refresh updates."};
     }
   } else {
     throw std::runtime_error{
         "Invalid boolean value specified for \"" + ALLOW_FILE_ROLL_OFF_KEY +
         "\" foreign table option. Value must be either 'true' or 'false'."};
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateServerOptions ( const ForeignServer * foreign_server ) const

overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters

foreign_server - foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 28 of file AbstractFileStorageDataWrapper.cpp.

References shared::contains(), g_enable_s3_fsi, join(), foreign_storage::OptionsContainer::options, S3_STORAGE_TYPE, STORAGE_TYPE_KEY, supported_server_options_, and supported_storage_types.

                                                {
   const auto& options = foreign_server->options;
   for (const auto& entry : options) {
     if (!shared::contains(supported_server_options_, entry.first)) {
       throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
                                "\". Option must be one of the following: " +
                                join(supported_server_options_, ", ") + "."};
     }
   }
 
   if (options.find(STORAGE_TYPE_KEY) == options.end()) {
     throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
                              "\"."};
   }
   const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
   if (!shared::contains(supported_storage_types, storage_type)) {
     throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
                              "\" option value. Value must be one of the following: " +
                              join(supported_storage_types, ", ") + "."};
   }
 
   if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
     throw std::runtime_error{
         "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
         "\" is not allowed because FSI S3 support is currently disabled."};
   }
 }

Here is the call graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions ( const ForeignTable * foreign_table ) const

overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters

foreign_table - foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 57 of file AbstractFileStorageDataWrapper.cpp.

References getFilePathOptions(), shared::validate_sort_options(), validateFilePath(), validateFilePathOptionKey(), and validateFileRollOffOption().

Referenced by foreign_storage::CsvDataWrapper::validateTableOptions(), and foreign_storage::RegexParserDataWrapper::validateTableOptions().

                                              {
   validateFilePathOptionKey(foreign_table);
   validateFilePath(foreign_table);
   shared::validate_sort_options(getFilePathOptions(foreign_table));
   validateFileRollOffOption(foreign_table);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateUserMappingOptions	(	const UserMapping *	user_mapping,
		const ForeignServer *	foreign_server
	)		const

overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters

user_mapping - user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 70 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignServer::data_wrapper_type.

                                                {
   throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
                            "\" data wrapper can only be created for AWS S3 backed "
                            "foreign servers. AWS S3 support is currently disabled."};
 }

Member Data Documentation

const std::string foreign_storage::AbstractFileStorageDataWrapper::ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"

inlinestatic

Definition at line 38 of file AbstractFileStorageDataWrapper.h.

Referenced by allowFileRollOff(), foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), getAlterableTableOptions(), and validateFileRollOffOption().

const std::string foreign_storage::AbstractFileStorageDataWrapper::BASE_PATH_KEY = "BASE_PATH"

inlinestatic

Definition at line 31 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), Catalog_Namespace::Catalog::initializeSystemServers(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_PATH_KEY = "FILE_PATH"

inlinestatic

Definition at line 32 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY

inlinestatic

Definition at line 36 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and getFilePathOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY

inlinestatic

Definition at line 37 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and getFilePathOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"

inlinestatic

Definition at line 34 of file AbstractFileStorageDataWrapper.h.

const std::string foreign_storage::AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"

inlinestatic

Definition at line 33 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and getFilePathOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE = "AWS_S3"

inlinestatic

Definition at line 35 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::Csv::validate_and_get_is_s3_select(), foreign_storage::CsvDataWrapper::validateAndGetIsS3Select(), and validateServerOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY = "STORAGE_TYPE"

inlinestatic

Definition at line 30 of file AbstractFileStorageDataWrapper.h.

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_server_options_

staticprivate

Initial value:

{STORAGE_TYPE_KEY,

BASE_PATH_KEY}

Definition at line 63 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::array<std::string, 1> foreign_storage::AbstractFileStorageDataWrapper::supported_storage_types

inlinestatic

Initial value:

{

LOCAL_FILE_STORAGE_TYPE}

Definition at line 41 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_table_options_

staticprivate

Initial value:

{
    FILE_PATH_KEY,
    REGEX_PATH_FILTER_KEY,
    FILE_SORT_ORDER_BY_KEY,
    FILE_SORT_REGEX_KEY,
    ALLOW_FILE_ROLL_OFF_KEY}

Definition at line 62 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedTableOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_user_mapping_options_ {}

staticprivate

Definition at line 64 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedUserMappingOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::THREADS_KEY = "THREADS"

inlinestatic

Definition at line 39 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), foreign_storage::get_num_threads(), foreign_storage::CsvFileBufferParser::validateAndGetCopyParams(), and foreign_storage::RegexFileBufferParser::validateAndGetCopyParams().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/AbstractFileStorageDataWrapper.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/AbstractFileStorageDataWrapper.cpp

Public Member Functions

Static Public Member Functions

Static Public Attributes

Static Protected Member Functions

Static Private Member Functions

Static Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation