OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Public Member Functions | Static Public Member Functions | Protected Member Functions | Private Member Functions | Static Private Attributes | List of all members
foreign_storage::CsvDataWrapper Class Reference

#include <CsvDataWrapper.h>

+ Inheritance diagram for foreign_storage::CsvDataWrapper:
+ Collaboration diagram for foreign_storage::CsvDataWrapper:

Public Member Functions

 CsvDataWrapper ()
 
 CsvDataWrapper (const int db_id, const ForeignTable *foreign_table)
 
 CsvDataWrapper (const int db_id, const ForeignTable *foreign_table, const UserMapping *user_mapping, const bool disable_cache=false)
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 		getSupportedTableOptions () const override
 
- Public Member Functions inherited from foreign_storage::AbstractTextFileDataWrapper
 AbstractTextFileDataWrapper ()
 
 AbstractTextFileDataWrapper (const int db_id, const ForeignTable *foreign_table)
 
 AbstractTextFileDataWrapper (const int db_id, const ForeignTable *foreign_table, const UserMapping *user_mapping, const bool disable_cache)
 
void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector) override
 
void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer) override
 
std::string getSerializedDataWrapper () const override
 
void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata) override
 
bool isRestored () const override
 
ParallelismLevel getCachedParallelismLevel () const override
 
ParallelismLevel getNonCachedParallelismLevel () const override
 
void createRenderGroupAnalyzers () override
 Create RenderGroupAnalyzers for poly columns. More...
 
bool isLazyFragmentFetchingEnabled () const override
 
- Public Member Functions inherited from foreign_storage::AbstractFileStorageDataWrapper
 AbstractFileStorageDataWrapper ()
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 		getSupportedUserMappingOptions () const override
 
const std::set< std::string > getAlterableTableOptions () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 

Static Public Member Functions

static bool validateAndGetIsS3Select (const ForeignTable *foreign_table)
 
- Static Public Member Functions inherited from foreign_storage::AbstractFileStorageDataWrapper
static shared::FilePathOptions getFilePathOptions (const ForeignTable *foreign_table)
 

Protected Member Functions

const TextFileBufferParser & getFileBufferParser () const override
 
- Protected Member Functions inherited from foreign_storage::AbstractTextFileDataWrapper
virtual std::optional< size_t > getMaxFileCount () const
 

Private Member Functions

std::set< std::string_view > getAllCsvTableOptions () const
 

Static Private Attributes

static const std::set
< std::string_view > 		csv_table_options_
 
static const CsvFileBufferParser csv_file_buffer_parser_ {}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 
- Static Public Attributes inherited from foreign_storage::AbstractFileStorageDataWrapper
static const std::string STORAGE_TYPE_KEY = "STORAGE_TYPE"
 
static const std::string BASE_PATH_KEY = "BASE_PATH"
 
static const std::string FILE_PATH_KEY = "FILE_PATH"
 
static const std::string REGEX_PATH_FILTER_KEY = "REGEX_PATH_FILTER"
 
static const std::string LOCAL_FILE_STORAGE_TYPE = "LOCAL_FILE"
 
static const std::string S3_STORAGE_TYPE = "AWS_S3"
 
static const std::string FILE_SORT_ORDER_BY_KEY = shared::FILE_SORT_ORDER_BY_KEY
 
static const std::string FILE_SORT_REGEX_KEY = shared::FILE_SORT_REGEX_KEY
 
static const std::string ALLOW_FILE_ROLL_OFF_KEY = "ALLOW_FILE_ROLL_OFF"
 
static const std::string THREADS_KEY = "THREADS"
 
static const std::array
< std::string, 1 > 		supported_storage_types
 
- Static Protected Member Functions inherited from foreign_storage::AbstractFileStorageDataWrapper
static std::string getFullFilePath (const ForeignTable *foreign_table)
 Returns the path to the source file/dir of the table. Depending on options this may result from a concatenation of server and table path options. More...
 
static bool allowFileRollOff (const ForeignTable *foreign_table)
 

Detailed Description

Definition at line 23 of file CsvDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::CsvDataWrapper::CsvDataWrapper ( )

Definition at line 23 of file CsvDataWrapper.cpp.

23 : AbstractTextFileDataWrapper() {}
foreign_storage::AbstractTextFileDataWrapper::AbstractTextFileDataWrapper
AbstractTextFileDataWrapper()
Definition: AbstractTextFileDataWrapper.cpp:37
foreign_storage::CsvDataWrapper::CsvDataWrapper ( const int  db_id,
const ForeignTable *  foreign_table 
)

Definition at line 25 of file CsvDataWrapper.cpp.

26  : AbstractTextFileDataWrapper(db_id, foreign_table) {}
foreign_storage::AbstractTextFileDataWrapper::AbstractTextFileDataWrapper
AbstractTextFileDataWrapper()
Definition: AbstractTextFileDataWrapper.cpp:37
foreign_storage::CsvDataWrapper::CsvDataWrapper ( const int  db_id,
const ForeignTable *  foreign_table,
const UserMapping *  user_mapping,
const bool  disable_cache = false 
)

Definition at line 28 of file CsvDataWrapper.cpp.

32  : AbstractTextFileDataWrapper(db_id, foreign_table, user_mapping, disable_cache) {}
foreign_storage::AbstractTextFileDataWrapper::AbstractTextFileDataWrapper
AbstractTextFileDataWrapper()
Definition: AbstractTextFileDataWrapper.cpp:37

Member Function Documentation

std::set< std::string_view > foreign_storage::CsvDataWrapper::getAllCsvTableOptions ( ) const
private

Definition at line 45 of file CsvDataWrapper.cpp.

References csv_table_options_, and foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions().

Referenced by getSupportedTableOptions().

45  {
46  std::set<std::string_view> supported_table_options(
47  AbstractFileStorageDataWrapper::getSupportedTableOptions().begin(),
48  AbstractFileStorageDataWrapper::getSupportedTableOptions().end());
49  supported_table_options.insert(csv_table_options_.begin(), csv_table_options_.end());
50  return supported_table_options;
51 }
foreign_storage::CsvDataWrapper::csv_table_options_
static const std::set< std::string_view > csv_table_options_
Definition: CsvDataWrapper.h:45
foreign_storage::AbstractFileStorageDataWrapper::getSupportedTableOptions
const std::set< std::string_view > & getSupportedTableOptions() const override
Definition: AbstractFileStorageDataWrapper.cpp:66

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const TextFileBufferParser & foreign_storage::CsvDataWrapper::getFileBufferParser ( ) const
overrideprotectedvirtual

Implements foreign_storage::AbstractTextFileDataWrapper.

Definition at line 53 of file CsvDataWrapper.cpp.

References csv_file_buffer_parser_.

53  {
54  return csv_file_buffer_parser_;
55 }
foreign_storage::CsvDataWrapper::csv_file_buffer_parser_
static const CsvFileBufferParser csv_file_buffer_parser_
Definition: CsvDataWrapper.h:46
const std::set< std::string_view > & foreign_storage::CsvDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Reimplemented from foreign_storage::AbstractFileStorageDataWrapper.

Definition at line 40 of file CsvDataWrapper.cpp.

References getAllCsvTableOptions().

40  {
41  static const auto supported_table_options = getAllCsvTableOptions();
42  return supported_table_options;
43 }
foreign_storage::CsvDataWrapper::getAllCsvTableOptions
std::set< std::string_view > getAllCsvTableOptions() const
Definition: CsvDataWrapper.cpp:45

+ Here is the call graph for this function:

bool foreign_storage::CsvDataWrapper::validateAndGetIsS3Select ( const ForeignTable *  foreign_table)
static

Definition at line 57 of file CsvDataWrapper.cpp.

References foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::options, foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

Referenced by foreign_storage::ForeignDataWrapperFactory::create(), foreign_storage::ForeignDataWrapperFactory::createForValidation(), and validateTableOptions().

57  {
58  static constexpr const char* S3_DIRECT = "S3_DIRECT";
59  static constexpr const char* S3_SELECT = "S3_SELECT";
60  static constexpr const char* S3_ACCESS_TYPE = "S3_ACCESS_TYPE";
61  auto access_type = foreign_table->options.find(S3_ACCESS_TYPE);
62 
63  if (access_type != foreign_table->options.end()) {
64  auto& server_options = foreign_table->foreign_server->options;
65  if (server_options.find(AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY)->second !=
66  AbstractFileStorageDataWrapper::S3_STORAGE_TYPE) {
67  throw std::runtime_error{
68  "The \"" + std::string{S3_ACCESS_TYPE} +
69  "\" option is only valid for foreign tables using servers with \"" +
70  AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY + "\" option value of \"" +
71  AbstractFileStorageDataWrapper::S3_STORAGE_TYPE + "\"."};
72  }
73  if (access_type->second != S3_DIRECT && access_type->second != S3_SELECT) {
74  throw std::runtime_error{
75  "Invalid value provided for the \"" + std::string{S3_ACCESS_TYPE} +
76  "\" option. Value must be one of the following: " + S3_DIRECT + ", " +
77  S3_SELECT + "."};
78  }
79  return (access_type->second == S3_SELECT);
80  } else {
81  return false;
82  }
83 }
foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY
static const std::string STORAGE_TYPE_KEY
Definition: AbstractFileStorageDataWrapper.h:30
foreign_storage::AbstractFileStorageDataWrapper::S3_STORAGE_TYPE
static const std::string S3_STORAGE_TYPE
Definition: AbstractFileStorageDataWrapper.h:35

+ Here is the caller graph for this function:

void foreign_storage::CsvDataWrapper::validateTableOptions ( const ForeignTable *  foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Reimplemented from foreign_storage::AbstractFileStorageDataWrapper.

Definition at line 34 of file CsvDataWrapper.cpp.

References csv_file_buffer_parser_, foreign_storage::CsvFileBufferParser::validateAndGetCopyParams(), validateAndGetIsS3Select(), and foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions().

34  {
35  AbstractTextFileDataWrapper::validateTableOptions(foreign_table);
36  csv_file_buffer_parser_.validateAndGetCopyParams(foreign_table);
37  validateAndGetIsS3Select(foreign_table);
38 }
foreign_storage::CsvDataWrapper::validateAndGetIsS3Select
static bool validateAndGetIsS3Select(const ForeignTable *foreign_table)
Definition: CsvDataWrapper.cpp:57
foreign_storage::AbstractFileStorageDataWrapper::validateTableOptions
void validateTableOptions(const ForeignTable *foreign_table) const override
Definition: AbstractFileStorageDataWrapper.cpp:57
foreign_storage::CsvFileBufferParser::validateAndGetCopyParams
import_export::CopyParams validateAndGetCopyParams(const ForeignTable *foreign_table) const override
Definition: CsvFileBufferParser.cpp:337
foreign_storage::CsvDataWrapper::csv_file_buffer_parser_
static const CsvFileBufferParser csv_file_buffer_parser_
Definition: CsvDataWrapper.h:46

+ Here is the call graph for this function:

Member Data Documentation

const CsvFileBufferParser foreign_storage::CsvDataWrapper::csv_file_buffer_parser_ {}
staticprivate

Definition at line 46 of file CsvDataWrapper.h.

Referenced by getFileBufferParser(), and validateTableOptions().

const std::set< std::string_view > foreign_storage::CsvDataWrapper::csv_table_options_
staticprivate
Initial value:
{"ARRAY_DELIMITER",
"ARRAY_MARKER",
"BUFFER_SIZE",
"DELIMITER",
"ESCAPE",
"HEADER",
"LINE_DELIMITER",
"LONLAT",
"NULLS",
"QUOTE",
"QUOTED",
"S3_ACCESS_TYPE",
"TRIM_SPACES"}

Definition at line 45 of file CsvDataWrapper.h.

Referenced by getAllCsvTableOptions().

The documentation for this class was generated from the following files: