OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Public Member Functions | Static Public Member Functions | Protected Member Functions | Private Member Functions | Static Private Attributes | List of all members
foreign_storage::CsvDataWrapper Class Reference

#include <CsvDataWrapper.h>

+ Inheritance diagram for foreign_storage::CsvDataWrapper:
+ Collaboration diagram for foreign_storage::CsvDataWrapper:

Public Member Functions

 CsvDataWrapper ()
 
 CsvDataWrapper (const int db_id, const ForeignTable *foreign_table)
 
 CsvDataWrapper (const int db_id, const ForeignTable *foreign_table, const UserMapping *user_mapping, const bool disable_cache=false)
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 		getSupportedTableOptions () const override
 
- Public Member Functions inherited from foreign_storage::AbstractTextFileDataWrapper
 AbstractTextFileDataWrapper ()
 
 AbstractTextFileDataWrapper (const int db_id, const ForeignTable *foreign_table)
 
 AbstractTextFileDataWrapper (const int db_id, const ForeignTable *foreign_table, const UserMapping *user_mapping, const bool disable_cache)
 
void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector) override
 
void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer) override
 
std::string getSerializedDataWrapper () const override
 
void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata) override
 
bool isRestored () const override
 
ParallelismLevel getCachedParallelismLevel () const override
 
ParallelismLevel getNonCachedParallelismLevel () const override
 
bool isLazyFragmentFetchingEnabled () const override
 
- Public Member Functions inherited from foreign_storage::AbstractFileStorageDataWrapper
 AbstractFileStorageDataWrapper ()
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 		getSupportedUserMappingOptions () const override
 
const std::set< std::string > getAlterableTableOptions () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 

Static Public Member Functions

static bool validateAndGetIsS3Select (const ForeignTable *foreign_table)
 
- Static Public Member Functions inherited from foreign_storage::AbstractFileStorageDataWrapper
static shared::FilePathOptions getFilePathOptions (const ForeignTable *foreign_table)
 

Protected Member Functions

const TextFileBufferParser & getFileBufferParser () const override
 
- Protected Member Functions inherited from foreign_storage::AbstractTextFileDataWrapper
virtual std::optional< size_t > getMaxFileCount () const
 

Private Member Functions

std::set< std::string_view > getAllCsvTableOptions () const
 

Static Private Attributes

static const std::set
< std::string_view > 		csv_table_options_
 
static const CsvFileBufferParser csv_file_buffer_parser_ {}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 
- Static Public Attributes inherited from foreign_storage::AbstractFileStorageDataWrapper
static const std::string STORAGE_TYPE_KEY = "STORAGE_TYPE"
 
static const std::string BASE_PATH_KEY = "BASE_PATH"
 
static const std::string FILE_PATH_KEY = "FILE_PATH"
 
static const std::string REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
 
static const std::string LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"
 
static const std::string S3_STORAGE_TYPE = "AWS_S3"
 
static const std::string FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
 
static const std::string FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
 
static const std::string ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"
 
static const std::string THREADS_KEY = "THREADS"
 
static const std::array
< std::string, 1 > 		supported_storage_types
 
- Static Protected Member Functions inherited from foreign_storage::AbstractFileStorageDataWrapper
static std::string getFullFilePath (const ForeignTable *foreign_table)
 Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...
 
static bool allowFileRollOff (const ForeignTable *foreign_table)
 

Detailed Description

Definition at line 23 of file CsvDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::CsvDataWrapper::CsvDataWrapper ( )

Definition at line 23 of file CsvDataWrapper.cpp.

23 : AbstractTextFileDataWrapper() {}
foreign_storage::AbstractTextFileDataWrapper::AbstractTextFileDataWrapper
AbstractTextFileDataWrapper()
Definition: AbstractTextFileDataWrapper.cpp:36
foreign_storage::CsvDataWrapper::CsvDataWrapper ( const int  db_id,
const ForeignTable *  foreign_table 
)

Definition at line 25 of file CsvDataWrapper.cpp.

26  : AbstractTextFileDataWrapper(db_id, foreign_table) {}
foreign_storage::AbstractTextFileDataWrapper::AbstractTextFileDataWrapper
AbstractTextFileDataWrapper()
Definition: AbstractTextFileDataWrapper.cpp:36
foreign_storage::CsvDataWrapper::CsvDataWrapper ( const int  db_id,
const ForeignTable *  foreign_table,
const UserMapping *  user_mapping,
const bool  disable_cache = false 
)

Definition at line 28 of file CsvDataWrapper.cpp.

32  : AbstractTextFileDataWrapper(db_id, foreign_table, user_mapping, disable_cache) {}
foreign_storage::AbstractTextFileDataWrapper::AbstractTextFileDataWrapper
AbstractTextFileDataWrapper()
Definition: AbstractTextFileDataWrapper.cpp:36

Member Function Documentation

std::set< std::string_view > foreign_storage::CsvDataWrapper::getAllCsvTableOptions ( ) const
private

Definition at line 45 of file CsvDataWrapper.cpp.

References csv_table_options_, and foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions().

Referenced by getSupportedTableOptions().

45  {
46  std::set<std::string_view> supported_table_options(
47  AbstractFileStorageDataWrapper::getSupportedTableOptions().begin(),
48  AbstractFileStorageDataWrapper::getSupportedTableOptions().end());
49  supported_table_options.insert(csv_table_options_.begin(), csv_table_options_.end());
50  return supported_table_options;
51 }
foreign_storage::CsvDataWrapper::csv_table_options_
static const std::set< std::string_view > csv_table_options_
Definition: CsvDataWrapper.h:45
foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions
const std::set< std::string_view > & getSupportedTableOptions() const override
Definition: AbstractFileStorageDataWrapper.cpp:66

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const TextFileBufferParser & foreign_storage::CsvDataWrapper::getFileBufferParser ( ) const
overrideprotectedvirtual

Implements foreign_storage::AbstractTextFileDataWrapper.

Definition at line 53 of file CsvDataWrapper.cpp.

References csv_file_buffer_parser_.

53  {
54  return csv_file_buffer_parser_;
55 }
foreign_storage::CsvDataWrapper::csv_file_buffer_parser_
static const CsvFileBufferParser csv_file_buffer_parser_
Definition: CsvDataWrapper.h:46
const std::set< std::string_view > & foreign_storage::CsvDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Reimplemented from foreign_storage::AbstractFileStorageDataWrapper.

Definition at line 40 of file CsvDataWrapper.cpp.

References getAllCsvTableOptions().

40  {
41  static const auto supported_table_options = getAllCsvTableOptions();
42  return supported_table_options;
43 }
foreign_storage::CsvDataWrapper::getAllCsvTableOptions
std::set< std::string_view > getAllCsvTableOptions() const
Definition: CsvDataWrapper.cpp:45

+ Here is the call graph for this function:

bool foreign_storage::CsvDataWrapper::validateAndGetIsS3Select ( const ForeignTable *  foreign_table)
static

Definition at line 57 of file CsvDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::options, foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

Referenced by foreign_storage::ForeignDataWrapperFactory::create(), foreign_storage::ForeignDataWrapperFactory::createForValidation(), and validateTableOptions().

57  {
58  static constexpr const char* S3_DIRECT = "S3_DIRECT";
59  static constexpr const char* S3_SELECT = "S3_SELECT";
60  static constexpr const char* S3_ACCESS_TYPE = "S3_ACCESS_TYPE";
61  auto access_type = foreign_table->options.find(S3_ACCESS_TYPE);
62 
63  if (access_type != foreign_table->options.end()) {
64  auto& server_options = foreign_table->foreign_server->options;
65  if (server_options.find(AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY)->second !=
66  AbstractFileStorageDataWrapper::S3_STORAGE_TYPE) {
67  throw std::runtime_error{
68  "The \"" + std::string{S3_ACCESS_TYPE} +
69  "\" option is only valid for foreign tables using servers with \"" +
70  AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY + "\" option value of \"" +
71  AbstractFileStorageDataWrapper::S3_STORAGE_TYPE + "\"."};
72  }
73  if (access_type->second != S3_DIRECT && access_type->second != S3_SELECT) {
74  throw std::runtime_error{
75  "Invalid value provided for the \"" + std::string{S3_ACCESS_TYPE} +
76  "\" option. Value must be one of the following: " + S3_DIRECT + ", " +
77  S3_SELECT + "."};
78  }
79  return (access_type->second == S3_SELECT);
80  } else {
81  return false;
82  }
83 }
foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY
static const std::string STORAGE_TYPE_KEY
Definition: AbstractFileStorageDataWrapper.h:30
foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE
static const std::string S3_STORAGE_TYPE
Definition: AbstractFileStorageDataWrapper.h:35

+ Here is the caller graph for this function:

void foreign_storage::CsvDataWrapper::validateTableOptions ( const ForeignTable *  foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Reimplemented from foreign_storage::AbstractFileStorageDataWrapper.

Definition at line 34 of file CsvDataWrapper.cpp.

References csv_file_buffer_parser_, foreign_storage::CsvFileBufferParser::validateAndGetCopyParams(), validateAndGetIsS3Select(), and foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions().

34  {
35  AbstractTextFileDataWrapper::validateTableOptions(foreign_table);
36  csv_file_buffer_parser_.validateAndGetCopyParams(foreign_table);
37  validateAndGetIsS3Select(foreign_table);
38 }
foreign_storage::CsvDataWrapper::validateAndGetIsS3Select
static bool validateAndGetIsS3Select(const ForeignTable *foreign_table)
Definition: CsvDataWrapper.cpp:57
foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions
void validateTableOptions(const ForeignTable *foreign_table) const override
Definition: AbstractFileStorageDataWrapper.cpp:57
foreign_storage::CsvFileBufferParser::validateAndGetCopyParams
import_export::CopyParams validateAndGetCopyParams(const ForeignTable *foreign_table) const override
Definition: CsvFileBufferParser.cpp:334
foreign_storage::CsvDataWrapper::csv_file_buffer_parser_
static const CsvFileBufferParser csv_file_buffer_parser_
Definition: CsvDataWrapper.h:46

+ Here is the call graph for this function:

Member Data Documentation

const CsvFileBufferParser foreign_storage::CsvDataWrapper::csv_file_buffer_parser_ {}
staticprivate

Definition at line 46 of file CsvDataWrapper.h.

Referenced by getFileBufferParser(), and validateTableOptions().

const std::set< std::string_view > foreign_storage::CsvDataWrapper::csv_table_options_
staticprivate
Initial value:
{"ARRAY_DELIMITER",
"ARRAY_MARKER",
"BUFFER_SIZE",
"DELIMITER",
"ESCAPE",
"HEADER",
"LINE_DELIMITER",
"LONLAT",
"NULLS",
"QUOTE",
"QUOTED",
"S3_ACCESS_TYPE",
"TRIM_SPACES"}

Definition at line 45 of file CsvDataWrapper.h.

Referenced by getAllCsvTableOptions().

The documentation for this class was generated from the following files: