OmniSciDB  94e8789169
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::Csv Namespace Reference

Namespaces

 anonymous_namespace{CsvShared.cpp}
 

Functions

bool validate_and_get_is_s3_select (const ForeignTable *foreign_table)
 
void validate_options (const ForeignTable *foreign_table)
 
void validate_file_path (const ForeignTable *foreign_table)
 
import_export::CopyParams validate_and_get_copy_params (const ForeignTable *foreign_table)
 
std::unique_ptr
< ForeignDataWrapper
get_csv_data_wrapper (int db_id, const ForeignTable *foreign_table)
 

Function Documentation

std::unique_ptr< ForeignDataWrapper > foreign_storage::Csv::get_csv_data_wrapper ( int  db_id,
const ForeignTable *  foreign_table 
)

Definition at line 158 of file CsvShared.cpp.

References UNREACHABLE, and validate_and_get_is_s3_select().

160  {
161  if (validate_and_get_is_s3_select(foreign_table)) {
162  UNREACHABLE();
163  return nullptr;
164  } else {
165  return std::make_unique<CsvDataWrapper>(db_id, foreign_table);
166  }
167 }
#define UNREACHABLE()
Definition: Logger.h:241
bool validate_and_get_is_s3_select(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:59

+ Here is the call graph for this function:

import_export::CopyParams foreign_storage::Csv::validate_and_get_copy_params ( const ForeignTable *  foreign_table)

Definition at line 101 of file CsvShared.cpp.

References import_export::HAS_HEADER, import_export::NO_HEADER, foreign_storage::OptionsContainer::options, import_export::CopyParams::plain_text, foreign_storage::Csv::anonymous_namespace{CsvShared.cpp}::validate_and_get_bool_value(), and foreign_storage::Csv::anonymous_namespace{CsvShared.cpp}::validate_and_get_string_with_length().

Referenced by foreign_storage::CsvDataWrapper::populateChunkMetadata(), foreign_storage::CsvDataWrapper::populateChunks(), foreign_storage::CsvDataWrapper::restoreDataWrapperInternals(), and validate_options().

102  {
103  import_export::CopyParams copy_params{};
104  copy_params.plain_text = true;
105  if (const auto& value =
106  validate_and_get_string_with_length(foreign_table, "ARRAY_DELIMITER", 1);
107  !value.empty()) {
108  copy_params.array_delim = value[0];
109  }
110  if (const auto& value =
111  validate_and_get_string_with_length(foreign_table, "ARRAY_MARKER", 2);
112  !value.empty()) {
113  copy_params.array_begin = value[0];
114  copy_params.array_end = value[1];
115  }
116  if (auto it = foreign_table->options.find("BUFFER_SIZE");
117  it != foreign_table->options.end()) {
118  copy_params.buffer_size = std::stoi(it->second);
119  }
120  if (const auto& value =
121  validate_and_get_string_with_length(foreign_table, "DELIMITER", 1);
122  !value.empty()) {
123  copy_params.delimiter = value[0];
124  }
125  if (const auto& value = validate_and_get_string_with_length(foreign_table, "ESCAPE", 1);
126  !value.empty()) {
127  copy_params.escape = value[0];
128  }
129  auto has_header = validate_and_get_bool_value(foreign_table, "HEADER");
130  if (has_header.has_value()) {
131  if (has_header.value()) {
132  copy_params.has_header = import_export::ImportHeaderRow::HAS_HEADER;
133  } else {
134  copy_params.has_header = import_export::ImportHeaderRow::NO_HEADER;
135  }
136  }
137  if (const auto& value =
138  validate_and_get_string_with_length(foreign_table, "LINE_DELIMITER", 1);
139  !value.empty()) {
140  copy_params.line_delim = value[0];
141  }
142  copy_params.lonlat =
143  validate_and_get_bool_value(foreign_table, "LONLAT").value_or(copy_params.lonlat);
144 
145  if (auto it = foreign_table->options.find("NULLS");
146  it != foreign_table->options.end()) {
147  copy_params.null_str = it->second;
148  }
149  if (const auto& value = validate_and_get_string_with_length(foreign_table, "QUOTE", 1);
150  !value.empty()) {
151  copy_params.quote = value[0];
152  }
153  copy_params.quoted =
154  validate_and_get_bool_value(foreign_table, "QUOTED").value_or(copy_params.quoted);
155  return copy_params;
156 }
std::optional< bool > validate_and_get_bool_value(const ForeignTable *foreign_table, const std::string &option_name)
Definition: CsvShared.cpp:41
std::string validate_and_get_string_with_length(const ForeignTable *foreign_table, const std::string &option_name, const size_t expected_num_chars)
Definition: CsvShared.cpp:25

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool foreign_storage::Csv::validate_and_get_is_s3_select ( const ForeignTable *  foreign_table)

Definition at line 59 of file CsvShared.cpp.

References foreign_storage::ForeignTable::foreign_server, foreign_storage::OptionsContainer::options, foreign_storage::ForeignServer::S3_STORAGE_TYPE, and foreign_storage::ForeignServer::STORAGE_TYPE_KEY.

Referenced by get_csv_data_wrapper(), and validate_options().

59  {
60  static constexpr const char* S3_DIRECT = "S3_DIRECT";
61  static constexpr const char* S3_SELECT = "S3_SELECT";
62  static constexpr const char* S3_ACCESS_TYPE = "S3_ACCESS_TYPE";
63  auto access_type = foreign_table->options.find(S3_ACCESS_TYPE);
64 
65  if (access_type != foreign_table->options.end()) {
66  auto& server_options = foreign_table->foreign_server->options;
67  if (server_options.find(ForeignServer::STORAGE_TYPE_KEY)->second !=
68  ForeignServer::S3_STORAGE_TYPE) {
69  throw std::runtime_error{std::string{S3_ACCESS_TYPE} +
70  " option only valid for tables using servers with " +
71  std::string{ForeignServer::STORAGE_TYPE_KEY} + " = " +
72  std::string{ForeignServer::S3_STORAGE_TYPE} + "."};
73  }
74  if (access_type->second != S3_DIRECT && access_type->second != S3_SELECT) {
75  throw std::runtime_error{"Invalid value provided for the " +
76  std::string{S3_ACCESS_TYPE} + " option. Value must be \"" +
77  S3_DIRECT + "\" or \"" + S3_SELECT + "\"."};
78  }
79 
80  return (access_type->second == S3_SELECT);
81  } else {
82  return false;
83  }
84 }

+ Here is the caller graph for this function:

void foreign_storage::Csv::validate_file_path ( const ForeignTable *  foreign_table)

Definition at line 92 of file CsvShared.cpp.

References foreign_storage::ForeignTable::foreign_server, foreign_storage::ForeignTable::getFullFilePath(), ddl_utils::IMPORT, foreign_storage::ForeignServer::LOCAL_FILE_STORAGE_TYPE, foreign_storage::OptionsContainer::options, foreign_storage::ForeignServer::STORAGE_TYPE_KEY, and ddl_utils::validate_allowed_file_path().

Referenced by validate_options().

92  {
93  auto& server_options = foreign_table->foreign_server->options;
94  if (server_options.find(ForeignServer::STORAGE_TYPE_KEY)->second ==
95  ForeignServer::LOCAL_FILE_STORAGE_TYPE) {
96  ddl_utils::validate_allowed_file_path(foreign_table->getFullFilePath(),
98  }
99 }
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:613

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::Csv::validate_options ( const ForeignTable *  foreign_table)

Definition at line 86 of file CsvShared.cpp.

References validate_and_get_copy_params(), validate_and_get_is_s3_select(), and validate_file_path().

Referenced by foreign_storage::CsvDataWrapper::validateOptions().

86  {
87  validate_and_get_copy_params(foreign_table);
88  validate_and_get_is_s3_select(foreign_table);
89  validate_file_path(foreign_table);
90 }
import_export::CopyParams validate_and_get_copy_params(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:101
void validate_file_path(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:92
bool validate_and_get_is_s3_select(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:59

+ Here is the call graph for this function:

+ Here is the caller graph for this function: