OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
AbstractFileStorageDataWrapper.cpp
Go to the documentation of this file.
2 
3 #include <codecvt>
4 #include <locale>
5 
7 #include "Catalog/ForeignTable.h"
9 #include "Shared/misc.h"
10 #include "Shared/thread_count.h"
11 #include "Utils/DdlUtils.h"
12 
13 extern bool g_enable_s3_fsi;
14 
15 namespace foreign_storage {
16 
17 size_t get_num_threads(const ForeignTable& table) {
18  auto num_threads = 0;
20  opt.has_value()) {
21  num_threads = std::stoi(opt.value());
22  }
23  return import_export::num_import_threads(num_threads);
24 }
25 
27 
29  const ForeignServer* foreign_server) const {
30  const auto& options = foreign_server->options;
31  for (const auto& entry : options) {
32  if (!shared::contains(supported_server_options_, entry.first)) {
33  throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
34  "\". Option must be one of the following: " +
35  join(supported_server_options_, ", ") + "."};
36  }
37  }
38 
39  if (options.find(STORAGE_TYPE_KEY) == options.end()) {
40  throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
41  "\"."};
42  }
43  const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
44  if (!shared::contains(supported_storage_types, storage_type)) {
45  throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
46  "\" option value. Value must be one of the following: " +
47  join(supported_storage_types, ", ") + "."};
48  }
49 
50  if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
51  throw std::runtime_error{
52  "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
53  "\" is not allowed because FSI S3 support is currently disabled."};
54  }
55 }
56 
58  const ForeignTable* foreign_table) const {
59  validateFilePathOptionKey(foreign_table);
60  validateFilePath(foreign_table);
62  validateFileRollOffOption(foreign_table);
63 }
64 
65 const std::set<std::string_view>&
68 }
69 
71  const UserMapping* user_mapping,
72  const ForeignServer* foreign_server) const {
73  throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
74  "\" data wrapper can only be created for AWS S3 backed "
75  "foreign servers. AWS S3 support is currently disabled."};
76 }
77 
78 const std::set<std::string_view>&
81 }
82 
84  auto& server_options = foreign_table->foreign_server->options;
85  if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
88  }
89 }
90 
91 namespace {
92 std::string append_file_path(const std::optional<std::string>& base,
93  const std::optional<std::string>& subdirectory) {
94 #ifdef _WIN32
95  const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
96  using convert_type = std::codecvt_utf8<wchar_t>;
97  std::wstring_convert<convert_type, wchar_t> converter;
98  std::string separator = converter.to_bytes(str_to_cov);
99 #else
100  const std::string separator{boost::filesystem::path::preferred_separator};
101 #endif
102  return std::regex_replace(
103  (base ? *base + separator : "") + (subdirectory ? *subdirectory : ""),
104  std::regex{separator + "{2,}"},
105  separator);
106 }
107 } // namespace
108 
114  const ForeignTable* foreign_table) {
115  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
116  std::optional<std::string> base_path{};
117  auto foreign_server = foreign_table->foreign_server;
118  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
119  CHECK(storage_type);
120 
121 #ifdef _WIN32
122  const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
123  using convert_type = std::codecvt_utf8<wchar_t>;
124  std::wstring_convert<convert_type, wchar_t> converter;
125  std::string separator = converter.to_bytes(str_to_cov);
126 #else
127  const std::string separator{boost::filesystem::path::preferred_separator};
128 #endif
129  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
130  base_path = foreign_server->getOption(BASE_PATH_KEY);
131  }
132 
133  // If both base_path and file_path are present, then concatenate. Otherwise we are just
134  // taking the one as the path. One of the two must exist, or we have failed validation.
135  CHECK(file_path || base_path);
136  return append_file_path(base_path, file_path);
137 }
138 
139 namespace {
140 void throw_file_path_error(const std::string_view& missing_path,
141  const std::string& table_name,
142  const std::string_view& file_path_key) {
143  std::stringstream ss;
144  ss << "No file_path found for Foreign Table \"" << table_name
145  << "\". Table must have either set a \"" << file_path_key << "\" option, or its "
146  << "parent server must have set a \"" << missing_path << "\" option.";
147  throw std::runtime_error(ss.str());
148 }
149 } // namespace
150 
151 // A valid path is a concatenation of the file_path and the base_path (for local storage).
152 // One of the two must be present.
154  const ForeignTable* foreign_table) {
155  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
156  auto foreign_server = foreign_table->foreign_server;
157  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
158  CHECK(storage_type) << "No storage type found in parent server. Server \""
159  << foreign_server->name << "\" is not valid.";
160  if (!file_path) {
161  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
162  if (!foreign_server->getOption(BASE_PATH_KEY)) {
164  }
165  } else {
166  UNREACHABLE() << "Unknown foreign storage type.";
167  }
168  }
169 }
170 
172  const ForeignTable* foreign_table) {
173  return {
177 }
178 
179 namespace {
180 std::optional<bool> get_file_roll_off_value(const ForeignTable* foreign_table) {
181  auto option =
183  if (option.has_value()) {
184  if (to_upper(option.value()) == "TRUE") {
185  return true;
186  } else if (to_upper(option.value()) == "FALSE") {
187  return false;
188  } else {
189  return {};
190  }
191  }
192  return false;
193 }
194 } // namespace
195 
197  const ForeignTable* foreign_table) {
198  auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
199  if (allow_file_roll_off.has_value()) {
200  if (allow_file_roll_off.value() && !foreign_table->isAppendMode()) {
201  throw std::runtime_error{"The \"" + ALLOW_FILE_ROLL_OFF_KEY +
202  "\" option can only be set to 'true' for foreign tables "
203  "with append refresh updates."};
204  }
205  } else {
206  throw std::runtime_error{
207  "Invalid boolean value specified for \"" + ALLOW_FILE_ROLL_OFF_KEY +
208  "\" foreign table option. Value must be either 'true' or 'false'."};
209  }
210 }
211 
213  auto allow_file_roll_off = get_file_roll_off_value(foreign_table);
214  if (allow_file_roll_off.has_value()) {
215  return allow_file_roll_off.value();
216  } else {
217  auto option = foreign_table->getOption(ALLOW_FILE_ROLL_OFF_KEY);
218  CHECK(option.has_value());
219  UNREACHABLE() << "Unexpected " << ALLOW_FILE_ROLL_OFF_KEY
220  << " value: " << option.value();
221  return false;
222  }
223 }
224 
226  const {
227  return {ALLOW_FILE_ROLL_OFF_KEY};
228 }
229 
230 const std::set<std::string_view> AbstractFileStorageDataWrapper::supported_table_options_{
231  FILE_PATH_KEY,
232  REGEX_PATH_FILTER_KEY,
235  ALLOW_FILE_ROLL_OFF_KEY};
236 
237 const std::set<std::string_view>
239  BASE_PATH_KEY};
240 
241 const std::set<std::string_view>
243 } // namespace foreign_storage
void throw_file_path_error(const std::string_view &missing_path, const std::string &table_name, const std::string_view &file_path_key)
const std::set< std::string_view > & getSupportedUserMappingOptions() const override
bool contains(const T &container, const U &element)
Definition: misc.h:204
static void validateFilePathOptionKey(const ForeignTable *foreign_table)
static const std::array< std::string, 1 > supported_storage_types
static shared::FilePathOptions getFilePathOptions(const ForeignTable *foreign_table)
const std::string FILE_SORT_REGEX_KEY
std::string tableName
size_t get_num_threads(const ForeignTable &table)
void validate_sort_options(const FilePathOptions &options)
std::string join(T const &container, std::string const &delim)
#define UNREACHABLE()
Definition: Logger.h:338
void validateUserMappingOptions(const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
static const std::set< std::string_view > supported_server_options_
void validateTableOptions(const ForeignTable *foreign_table) const override
static const std::set< std::string_view > supported_user_mapping_options_
bool g_enable_s3_fsi
Definition: Catalog.cpp:97
const std::string FILE_SORT_ORDER_BY_KEY
static void validateFileRollOffOption(const ForeignTable *foreign_table)
std::string append_file_path(const std::optional< std::string > &base, const std::optional< std::string > &subdirectory)
std::optional< std::string > getOption(const std::string_view &key) const
size_t num_import_threads(const int32_t copy_params_threads)
Definition: thread_count.h:31
bool isAppendMode() const
Checks if the table is in append mode.
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:822
std::string to_upper(const std::string &str)
const std::set< std::string_view > & getSupportedTableOptions() const override
static bool allowFileRollOff(const ForeignTable *foreign_table)
std::optional< bool > get_file_roll_off_value(const ForeignTable *foreign_table)
void validateServerOptions(const ForeignServer *foreign_server) const override
const ForeignServer * foreign_server
Definition: ForeignTable.h:57
#define CHECK(condition)
Definition: Logger.h:291
static const std::set< std::string_view > supported_table_options_
const std::set< std::string > getAlterableTableOptions() const override
static std::string getFullFilePath(const ForeignTable *foreign_table)
Returns the path to the source file/dir of the table. Depending on options this may result from a con...
static void validateFilePath(const ForeignTable *foreign_table)