#include <AbstractFileStorageDataWrapper.h>

Inheritance diagram for foreign_storage::AbstractFileStorageDataWrapper:

Collaboration diagram for foreign_storage::AbstractFileStorageDataWrapper:

Public Member Functions
	AbstractFileStorageDataWrapper ()

void	validateServerOptions (const ForeignServer *foreign_server) const override

void	validateTableOptions (const ForeignTable *foreign_table) const override

const std::set < std::string_view > &	getSupportedTableOptions () const override

void	validateUserMappingOptions (const UserMapping user_mapping, const ForeignServer foreign_server) const override

const std::set < std::string_view > &	getSupportedUserMappingOptions () const override

const std::set< std::string >	getAlterableTableOptions () const override

Public Member Functions inherited from foreign_storage::ForeignDataWrapper
	ForeignDataWrapper ()=default

virtual	~ForeignDataWrapper ()=default

virtual void	populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector)=0

virtual void	populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer=nullptr)=0

virtual std::string	getSerializedDataWrapper () const =0

virtual void	restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata)=0

virtual bool	isRestored () const =0

virtual void	validateSchema (const std::list< ColumnDescriptor > &columns) const

virtual ParallelismLevel	getCachedParallelismLevel () const

virtual ParallelismLevel	getNonCachedParallelismLevel () const

virtual void	createRenderGroupAnalyzers ()
	Create RenderGroupAnalyzers for poly columns. More...

virtual bool	isLazyFragmentFetchingEnabled () const

Static Public Member Functions
static shared::FilePathOptions	getFilePathOptions (const ForeignTable *foreign_table)

Static Public Attributes
static const std::string	STORAGE_TYPE_KEY = "STORAGE_TYPE"

static const std::string	BASE_PATH_KEY = "BASE_PATH"

static const std::string	FILE_PATH_KEY = "FILE_PATH"

static const std::string	REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"

static const std::string	LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"

static const std::string	S3_STORAGE_TYPE = "AWS_S3"

static const std::string	FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY

static const std::string	FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY

static const std::string	ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"

static const std::string	THREADS_KEY = "THREADS"

static const std::array < std::string, 1 >	supported_storage_types

Static Protected Member Functions
static std::string	getFullFilePath (const ForeignTable *foreign_table)
	Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...

static bool	allowFileRollOff (const ForeignTable *foreign_table)

Static Private Member Functions
static void	validateFilePath (const ForeignTable *foreign_table)

static void	validateFilePathOptionKey (const ForeignTable *foreign_table)

static void	validateFileRollOffOption (const ForeignTable *foreign_table)

Static Private Attributes
static const std::set < std::string_view >	supported_table_options_

static const std::set < std::string_view >	supported_server_options_

static const std::set < std::string_view >	supported_user_mapping_options_ {}

Additional Inherited Members
Public Types inherited from foreign_storage::ForeignDataWrapper
enum	ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }

Detailed Description

Definition at line 11 of file AbstractFileStorageDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::AbstractFileStorageDataWrapper::AbstractFileStorageDataWrapper ( )

Definition at line 26 of file AbstractFileStorageDataWrapper.cpp.

26 {}

Member Function Documentation

bool foreign_storage::AbstractFileStorageDataWrapper::allowFileRollOff ( const ForeignTable * foreign_table )

staticprotected

Definition at line 212 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, CHECK, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), foreign_storage::OptionsContainer::getOption(), and UNREACHABLE.

Referenced by foreign_storage::ParquetDataWrapper::addNewFile(), foreign_storage::ParquetDataWrapper::fetchChunkMetadata(), foreign_storage::ParquetDataWrapper::isNewFile(), and foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                                                                        {
   auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
   if (allow_file_roll_off.has_value()) {
     return allow_file_roll_off.value();
   } else {
     auto option = foreign_table->getOption(ALLOW_FILE_ROLL_OFF_KEY);
     CHECK(option.has_value());
     UNREACHABLE() << "Unexpected " << ALLOW_FILE_ROLL_OFF_KEY
                   << " value: " << option.value();
     return false;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const std::set< std::string > foreign_storage::AbstractFileStorageDataWrapper::getAlterableTableOptions ( ) const

overridevirtual

Gets the subset of table options that can be altered for the data wrapper.

Reimplemented from foreign_storage::ForeignDataWrapper.

Definition at line 225 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY.

           {
   return {ALLOW_FILE_ROLL_OFF_KEY};
 }

shared::FilePathOptions foreign_storage::AbstractFileStorageDataWrapper::getFilePathOptions ( const ForeignTable * foreign_table )

static

Definition at line 171 of file AbstractFileStorageDataWrapper.cpp.

References FILE_SORT_ORDER_BY_KEY, FILE_SORT_REGEX_KEY, foreign_storage::OptionsContainer::getOption(), and REGEX_PATH_FILTER_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and validateTableOptions().

                                        {
   return {
       foreign_table->getOption(AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY),
       foreign_table->getOption(AbstractFileStorageDataWrapper::FILE_SORT_ORDER_BY_KEY),
       foreign_table->getOption(AbstractFileStorageDataWrapper::FILE_SORT_REGEX_KEY)};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::AbstractFileStorageDataWrapper::getFullFilePath ( const ForeignTable * foreign_table )

staticprotected

Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options.

Definition at line 113 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::append_file_path(), BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, generate_TableFunctionsFactory_init::separator, and STORAGE_TYPE_KEY.

Referenced by foreign_storage::ParquetDataWrapper::getAllFilePaths(), foreign_storage::ParquetImporter::getAllFilePaths(), foreign_storage::ParquetDataWrapper::getDataPreview(), foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), foreign_storage::AbstractTextFileDataWrapper::populateChunks(), foreign_storage::AbstractTextFileDataWrapper::restoreDataWrapperInternals(), and validateFilePath().

                                        {
   auto file_path = foreign_table->getOption(FILE_PATH_KEY);
   std::optional<std::string> base_path{};
   auto foreign_server = foreign_table->foreign_server;
   auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
   CHECK(storage_type);
 
 #ifdef _WIN32
   const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
   using convert_type = std::codecvt_utf8<wchar_t>;
   std::wstring_convert<convert_type, wchar_t> converter;
   std::string separator = converter.to_bytes(str_to_cov);
 #else
   const std::string separator{boost::filesystem::path::preferred_separator};
 #endif
   if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
     base_path = foreign_server->getOption(BASE_PATH_KEY);
   }
 
   // If both base_path and file_path are present, then concatenate.  Otherwise we are just
   // taking the one as the path.  One of the two must exist, or we have failed validation.
   CHECK(file_path || base_path);
   return append_file_path(base_path, file_path);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions ( ) const

overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 66 of file AbstractFileStorageDataWrapper.cpp.

References supported_table_options_.

Referenced by foreign_storage::CsvDataWrapper::getAllCsvTableOptions(), and foreign_storage::RegexParserDataWrapper::getAllRegexTableOptions().

                                                                {
   return supported_table_options_;
 }

Here is the caller graph for this function:

const std::set< std::string_view > & foreign_storage::AbstractFileStorageDataWrapper::getSupportedUserMappingOptions ( ) const

overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 79 of file AbstractFileStorageDataWrapper.cpp.

References supported_user_mapping_options_.

                                                                      {
   return supported_user_mapping_options_;
 }

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePath ( const ForeignTable * foreign_table )

staticprivate

Definition at line 83 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, getFullFilePath(), ddl_utils::IMPORT, LOCAL_FILE_STORAGE_TYPE, foreign_storage::OptionsContainer::options, STORAGE_TYPE_KEY, and ddl_utils::validate_allowed_file_path().

Referenced by validateTableOptions().

                                                                                        {
   auto& server_options = foreign_table->foreign_server->options;
   if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
     ddl_utils::validate_allowed_file_path(
         getFullFilePath(foreign_table), ddl_utils::DataTransferType::IMPORT, true);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFilePathOptionKey ( const ForeignTable * foreign_table )

staticprivate

Definition at line 153 of file AbstractFileStorageDataWrapper.cpp.

References BASE_PATH_KEY, CHECK, FILE_PATH_KEY, foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::getOption(), LOCAL_FILE_STORAGE_TYPE, STORAGE_TYPE_KEY, TableDescriptor::tableName, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::throw_file_path_error(), and UNREACHABLE.

Referenced by validateTableOptions().

                                        {
   auto file_path = foreign_table->getOption(FILE_PATH_KEY);
   auto foreign_server = foreign_table->foreign_server;
   auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
   CHECK(storage_type) << "No storage type found in parent server. Server \""
                       << foreign_server->name << "\" is not valid.";
   if (!file_path) {
     if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
       if (!foreign_server->getOption(BASE_PATH_KEY)) {
         throw_file_path_error(BASE_PATH_KEY, foreign_table->tableName, FILE_PATH_KEY);
       }
     } else {
       UNREACHABLE() << "Unknown foreign storage type.";
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateFileRollOffOption ( const ForeignTable * foreign_table )

staticprivate

Definition at line 196 of file AbstractFileStorageDataWrapper.cpp.

References ALLOW_FILE_ROLL_OFF_KEY, foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), and foreign_storage::ForeignTable::isAppendMode().

Referenced by validateTableOptions().

                                        {
   auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
   if (allow_file_roll_off.has_value()) {
     if (allow_file_roll_off.value() && !foreign_table->isAppendMode()) {
       throw std::runtime_error{"The \"" + ALLOW_FILE_ROLL_OFF_KEY +
                                "\" option can only be set to 'true' for foreign tables "
                                "with append refresh updates."};
     }
   } else {
     throw std::runtime_error{
         "Invalid boolean value specified for \"" + ALLOW_FILE_ROLL_OFF_KEY +
         "\" foreign table option. Value must be either 'true' or 'false'."};
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateServerOptions ( const ForeignServer * foreign_server ) const

overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters

foreign_server - foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 28 of file AbstractFileStorageDataWrapper.cpp.

References shared::contains(), g_enable_s3_fsi, join(), foreign_storage::OptionsContainer::options, S3_STORAGE_TYPE, STORAGE_TYPE_KEY, supported_server_options_, and supported_storage_types.

                                                {
   const auto& options = foreign_server->options;
   for (const auto& entry : options) {
     if (!shared::contains(supported_server_options_, entry.first)) {
       throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
                                "\". Option must be one of the following: " +
                                join(supported_server_options_, ", ") + "."};
     }
   }
 
   if (options.find(STORAGE_TYPE_KEY) == options.end()) {
     throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
                              "\"."};
   }
   const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
   if (!shared::contains(supported_storage_types, storage_type)) {
     throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
                              "\" option value. Value must be one of the following: " +
                              join(supported_storage_types, ", ") + "."};
   }
 
   if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
     throw std::runtime_error{
         "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
         "\" is not allowed because FSI S3 support is currently disabled."};
   }
 }

Here is the call graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions ( const ForeignTable * foreign_table ) const

overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters

foreign_table - foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Reimplemented in foreign_storage::CsvDataWrapper, and foreign_storage::RegexParserDataWrapper.

Definition at line 57 of file AbstractFileStorageDataWrapper.cpp.

References getFilePathOptions(), shared::validate_sort_options(), validateFilePath(), validateFilePathOptionKey(), and validateFileRollOffOption().

Referenced by foreign_storage::CsvDataWrapper::validateTableOptions(), and foreign_storage::RegexParserDataWrapper::validateTableOptions().

                                              {
   validateFilePathOptionKey(foreign_table);
   validateFilePath(foreign_table);
   shared::validate_sort_options(getFilePathOptions(foreign_table));
   validateFileRollOffOption(foreign_table);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::AbstractFileStorageDataWrapper::validateUserMappingOptions	(	const UserMapping *	user_mapping,
		const ForeignServer *	foreign_server
	)		const

overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters

user_mapping - user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 70 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::ForeignServer::data_wrapper_type.

                                                {
   throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
                            "\" data wrapper can only be created for AWS S3 backed "
                            "foreign servers. AWS S3 support is currently disabled."};
 }

Member Data Documentation

const std::string foreign_storage::AbstractFileStorageDataWrapper::ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"

inlinestatic

Definition at line 38 of file AbstractFileStorageDataWrapper.h.

Referenced by allowFileRollOff(), foreign_storage::anonymous_namespace{AbstractFileStorageDataWrapper.cpp}::get_file_roll_off_value(), getAlterableTableOptions(), and validateFileRollOffOption().

const std::string foreign_storage::AbstractFileStorageDataWrapper::BASE_PATH_KEY = "BASE_PATH"

inlinestatic

Definition at line 31 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), Catalog_Namespace::Catalog::initializeSystemServers(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_PATH_KEY = "FILE_PATH"

inlinestatic

Definition at line 32 of file AbstractFileStorageDataWrapper.h.

Referenced by getFullFilePath(), and validateFilePathOptionKey().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY

inlinestatic

Definition at line 36 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and getFilePathOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY

inlinestatic

Definition at line 37 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and getFilePathOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"

inlinestatic

Definition at line 34 of file AbstractFileStorageDataWrapper.h.

const std::string foreign_storage::AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"

inlinestatic

Definition at line 33 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and getFilePathOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE = "AWS_S3"

inlinestatic

Definition at line 35 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::Csv::validate_and_get_is_s3_select(), foreign_storage::CsvDataWrapper::validateAndGetIsS3Select(), and validateServerOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY = "STORAGE_TYPE"

inlinestatic

Definition at line 30 of file AbstractFileStorageDataWrapper.h.

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_server_options_

staticprivate

Initial value:

{STORAGE_TYPE_KEY,

BASE_PATH_KEY}

Definition at line 63 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::array<std::string, 1> foreign_storage::AbstractFileStorageDataWrapper::supported_storage_types

inlinestatic

Initial value:

{

LOCAL_FILE_STORAGE_TYPE}

Definition at line 41 of file AbstractFileStorageDataWrapper.h.

Referenced by validateServerOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_table_options_

staticprivate

Initial value:

{
    FILE_PATH_KEY,
    REGEX_PATH_FILTER_KEY,
    FILE_SORT_ORDER_BY_KEY,
    FILE_SORT_REGEX_KEY,
    ALLOW_FILE_ROLL_OFF_KEY}

Definition at line 62 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedTableOptions().

const std::set< std::string_view > foreign_storage::AbstractFileStorageDataWrapper::supported_user_mapping_options_ {}

staticprivate

Definition at line 64 of file AbstractFileStorageDataWrapper.h.

Referenced by getSupportedUserMappingOptions().

const std::string foreign_storage::AbstractFileStorageDataWrapper::THREADS_KEY = "THREADS"

inlinestatic

Definition at line 39 of file AbstractFileStorageDataWrapper.h.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), foreign_storage::get_num_threads(), foreign_storage::CsvFileBufferParser::validateAndGetCopyParams(), and foreign_storage::RegexFileBufferParser::validateAndGetCopyParams().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/AbstractFileStorageDataWrapper.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/AbstractFileStorageDataWrapper.cpp

Public Member Functions

Static Public Member Functions

Static Public Attributes

Static Protected Member Functions

Static Private Member Functions

Static Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation