OmniSciDB  94e8789169
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
CsvShared.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2021 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #include "CsvShared.h"
17 #include "CsvDataWrapper.h"
20 #include "Utils/DdlUtils.h"
21 
22 namespace foreign_storage {
23 namespace Csv {
24 namespace {
25 std::string validate_and_get_string_with_length(const ForeignTable* foreign_table,
26  const std::string& option_name,
27  const size_t expected_num_chars) {
28  if (auto it = foreign_table->options.find(option_name);
29  it != foreign_table->options.end()) {
30  if (it->second.length() != expected_num_chars) {
31  throw std::runtime_error{"Value of \"" + option_name +
32  "\" foreign table option has the wrong number of "
33  "characters. Expected " +
34  std::to_string(expected_num_chars) + " character(s)."};
35  }
36  return it->second;
37  }
38  return "";
39 }
40 
41 std::optional<bool> validate_and_get_bool_value(const ForeignTable* foreign_table,
42  const std::string& option_name) {
43  if (auto it = foreign_table->options.find(option_name);
44  it != foreign_table->options.end()) {
45  if (boost::iequals(it->second, "TRUE")) {
46  return true;
47  } else if (boost::iequals(it->second, "FALSE")) {
48  return false;
49  } else {
50  throw std::runtime_error{"Invalid boolean value specified for \"" + option_name +
51  "\" foreign table option. "
52  "Value must be either 'true' or 'false'."};
53  }
54  }
55  return std::nullopt;
56 }
57 } // namespace
58 
59 bool validate_and_get_is_s3_select(const ForeignTable* foreign_table) {
60  static constexpr const char* S3_DIRECT = "S3_DIRECT";
61  static constexpr const char* S3_SELECT = "S3_SELECT";
62  static constexpr const char* S3_ACCESS_TYPE = "S3_ACCESS_TYPE";
63  auto access_type = foreign_table->options.find(S3_ACCESS_TYPE);
64 
65  if (access_type != foreign_table->options.end()) {
66  auto& server_options = foreign_table->foreign_server->options;
67  if (server_options.find(ForeignServer::STORAGE_TYPE_KEY)->second !=
69  throw std::runtime_error{std::string{S3_ACCESS_TYPE} +
70  " option only valid for tables using servers with " +
71  std::string{ForeignServer::STORAGE_TYPE_KEY} + " = " +
72  std::string{ForeignServer::S3_STORAGE_TYPE} + "."};
73  }
74  if (access_type->second != S3_DIRECT && access_type->second != S3_SELECT) {
75  throw std::runtime_error{"Invalid value provided for the " +
76  std::string{S3_ACCESS_TYPE} + " option. Value must be \"" +
77  S3_DIRECT + "\" or \"" + S3_SELECT + "\"."};
78  }
79 
80  return (access_type->second == S3_SELECT);
81  } else {
82  return false;
83  }
84 }
85 
86 void validate_options(const ForeignTable* foreign_table) {
87  validate_and_get_copy_params(foreign_table);
88  validate_and_get_is_s3_select(foreign_table);
89  validate_file_path(foreign_table);
90 }
91 
92 void validate_file_path(const ForeignTable* foreign_table) {
93  auto& server_options = foreign_table->foreign_server->options;
94  if (server_options.find(ForeignServer::STORAGE_TYPE_KEY)->second ==
98  }
99 }
100 
102  const ForeignTable* foreign_table) {
103  import_export::CopyParams copy_params{};
104  copy_params.plain_text = true;
105  if (const auto& value =
106  validate_and_get_string_with_length(foreign_table, "ARRAY_DELIMITER", 1);
107  !value.empty()) {
108  copy_params.array_delim = value[0];
109  }
110  if (const auto& value =
111  validate_and_get_string_with_length(foreign_table, "ARRAY_MARKER", 2);
112  !value.empty()) {
113  copy_params.array_begin = value[0];
114  copy_params.array_end = value[1];
115  }
116  if (auto it = foreign_table->options.find("BUFFER_SIZE");
117  it != foreign_table->options.end()) {
118  copy_params.buffer_size = std::stoi(it->second);
119  }
120  if (const auto& value =
121  validate_and_get_string_with_length(foreign_table, "DELIMITER", 1);
122  !value.empty()) {
123  copy_params.delimiter = value[0];
124  }
125  if (const auto& value = validate_and_get_string_with_length(foreign_table, "ESCAPE", 1);
126  !value.empty()) {
127  copy_params.escape = value[0];
128  }
129  auto has_header = validate_and_get_bool_value(foreign_table, "HEADER");
130  if (has_header.has_value()) {
131  if (has_header.value()) {
132  copy_params.has_header = import_export::ImportHeaderRow::HAS_HEADER;
133  } else {
134  copy_params.has_header = import_export::ImportHeaderRow::NO_HEADER;
135  }
136  }
137  if (const auto& value =
138  validate_and_get_string_with_length(foreign_table, "LINE_DELIMITER", 1);
139  !value.empty()) {
140  copy_params.line_delim = value[0];
141  }
142  copy_params.lonlat =
143  validate_and_get_bool_value(foreign_table, "LONLAT").value_or(copy_params.lonlat);
144 
145  if (auto it = foreign_table->options.find("NULLS");
146  it != foreign_table->options.end()) {
147  copy_params.null_str = it->second;
148  }
149  if (const auto& value = validate_and_get_string_with_length(foreign_table, "QUOTE", 1);
150  !value.empty()) {
151  copy_params.quote = value[0];
152  }
153  copy_params.quoted =
154  validate_and_get_bool_value(foreign_table, "QUOTED").value_or(copy_params.quoted);
155  return copy_params;
156 }
157 
158 std::unique_ptr<ForeignDataWrapper> get_csv_data_wrapper(
159  int db_id,
160  const ForeignTable* foreign_table) {
161  if (validate_and_get_is_s3_select(foreign_table)) {
162  UNREACHABLE();
163  return nullptr;
164  } else {
165  return std::make_unique<CsvDataWrapper>(db_id, foreign_table);
166  }
167 }
168 
169 } // namespace Csv
170 
171 } // namespace foreign_storage
std::optional< bool > validate_and_get_bool_value(const ForeignTable *foreign_table, const std::string &option_name)
Definition: CsvShared.cpp:41
import_export::CopyParams validate_and_get_copy_params(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:101
static constexpr std::string_view S3_STORAGE_TYPE
Definition: ForeignServer.h:49
#define UNREACHABLE()
Definition: Logger.h:241
std::string to_string(char const *&&v)
void validate_options(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:86
static constexpr std::string_view STORAGE_TYPE_KEY
Definition: ForeignServer.h:45
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:613
std::string validate_and_get_string_with_length(const ForeignTable *foreign_table, const std::string &option_name, const size_t expected_num_chars)
Definition: CsvShared.cpp:25
std::unique_ptr< ForeignDataWrapper > get_csv_data_wrapper(int db_id, const ForeignTable *foreign_table)
Definition: CsvShared.cpp:158
static constexpr std::string_view LOCAL_FILE_STORAGE_TYPE
Definition: ForeignServer.h:47
void validate_file_path(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:92
std::string getFullFilePath() const
Returns the path to the source file/dir of the table. Depending on options this may result from a con...
const ForeignServer * foreign_server
Definition: ForeignTable.h:63
bool validate_and_get_is_s3_select(const ForeignTable *foreign_table)
Definition: CsvShared.cpp:59