OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
AbstractFileStorageDataWrapper.cpp
Go to the documentation of this file.
2 
3 #include <codecvt>
4 #include <locale>
5 
7 #include "Catalog/ForeignTable.h"
9 #include "Shared/misc.h"
10 #include "Utils/DdlUtils.h"
11 
12 extern bool g_enable_s3_fsi;
13 
14 namespace foreign_storage {
16 
18  const ForeignServer* foreign_server) const {
19  const auto& options = foreign_server->options;
20  for (const auto& entry : options) {
21  if (!shared::contains(supported_server_options_, entry.first)) {
22  throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
23  "\". Option must be one of the following: " +
24  join(supported_server_options_, ", ") + "."};
25  }
26  }
27 
28  if (options.find(STORAGE_TYPE_KEY) == options.end()) {
29  throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
30  "\"."};
31  }
32  const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
33  if (!shared::contains(supported_storage_types, storage_type)) {
34  throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
35  "\" option value. Value must be one of the following: " +
36  join(supported_storage_types, ", ") + "."};
37  }
38 
39  if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
40  throw std::runtime_error{
41  "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
42  "\" is not allowed because FSI S3 support is currently disabled."};
43  }
44 }
45 
47  const ForeignTable* foreign_table) const {
48  validateFilePathOptionKey(foreign_table);
49  validateFilePath(foreign_table);
51  foreign_table->getOption(FILE_SORT_REGEX_KEY));
52 }
53 
54 const std::set<std::string_view>&
57 }
58 
60  const UserMapping* user_mapping,
61  const ForeignServer* foreign_server) const {
62  throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
63  "\" data wrapper can only be created for AWS S3 backed "
64  "foreign servers. AWS S3 support is currently disabled."};
65 }
66 
67 const std::set<std::string_view>&
70 }
71 
73  auto& server_options = foreign_table->foreign_server->options;
74  if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
77  }
78 }
79 
80 namespace {
81 std::string append_file_path(const std::optional<std::string>& base,
82  const std::optional<std::string>& subdirectory) {
83 #ifdef _WIN32
84  const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
85  using convert_type = std::codecvt_utf8<wchar_t>;
86  std::wstring_convert<convert_type, wchar_t> converter;
87  std::string separator = converter.to_bytes(str_to_cov);
88 #else
89  const std::string separator{boost::filesystem::path::preferred_separator};
90 #endif
91  return std::regex_replace(
92  (base ? *base + separator : "") + (subdirectory ? *subdirectory : ""),
93  std::regex{separator + "{2,}"},
94  separator);
95 }
96 } // namespace
97 
103  const ForeignTable* foreign_table) {
104  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
105  std::optional<std::string> base_path{};
106  auto foreign_server = foreign_table->foreign_server;
107  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
108  CHECK(storage_type);
109 
110 #ifdef _WIN32
111  const std::wstring str_to_cov{boost::filesystem::path::preferred_separator};
112  using convert_type = std::codecvt_utf8<wchar_t>;
113  std::wstring_convert<convert_type, wchar_t> converter;
114  std::string separator = converter.to_bytes(str_to_cov);
115 #else
116  const std::string separator{boost::filesystem::path::preferred_separator};
117 #endif
118  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
119  base_path = foreign_server->getOption(BASE_PATH_KEY);
120  }
121 
122  // If both base_path and file_path are present, then concatenate. Otherwise we are just
123  // taking the one as the path. One of the two must exist, or we have failed validation.
124  CHECK(file_path || base_path);
125  return append_file_path(base_path, file_path);
126 }
127 
128 namespace {
129 void throw_file_path_error(const std::string_view& missing_path,
130  const std::string& table_name,
131  const std::string_view& file_path_key) {
132  std::stringstream ss;
133  ss << "No file_path found for Foreign Table \"" << table_name
134  << "\". Table must have either set a \"" << file_path_key << "\" option, or its "
135  << "parent server must have set a \"" << missing_path << "\" option.";
136  throw std::runtime_error(ss.str());
137 }
138 } // namespace
139 
140 // A valid path is a concatenation of the file_path and the base_path (for local storage).
141 // One of the two must be present.
143  const ForeignTable* foreign_table) {
144  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
145  auto foreign_server = foreign_table->foreign_server;
146  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
147  CHECK(storage_type) << "No storage type found in parent server. Server \""
148  << foreign_server->name << "\" is not valid.";
149  if (!file_path) {
150  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
151  if (!foreign_server->getOption(BASE_PATH_KEY)) {
153  }
154  } else {
155  UNREACHABLE() << "Unknown foreign storage type.";
156  }
157  }
158 }
159 
160 const std::set<std::string_view> AbstractFileStorageDataWrapper::supported_table_options_{
161  FILE_PATH_KEY,
162  REGEX_PATH_FILTER_KEY,
165 
166 const std::set<std::string_view>
168  BASE_PATH_KEY};
169 
170 const std::set<std::string_view>
172 } // namespace foreign_storage
void throw_file_path_error(const std::string_view &missing_path, const std::string &table_name, const std::string_view &file_path_key)
const std::set< std::string_view > & getSupportedUserMappingOptions() const override
bool contains(const T &container, const U &element)
Definition: misc.h:196
static void validateFilePathOptionKey(const ForeignTable *foreign_table)
static const std::array< std::string, 1 > supported_storage_types
const std::string FILE_SORT_REGEX_KEY
std::string tableName
void validate_sort_options(const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex)
std::string join(T const &container, std::string const &delim)
#define UNREACHABLE()
Definition: Logger.h:267
void validateUserMappingOptions(const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
static const std::set< std::string_view > supported_server_options_
void validateTableOptions(const ForeignTable *foreign_table) const override
static const std::set< std::string_view > supported_user_mapping_options_
bool g_enable_s3_fsi
Definition: Catalog.cpp:98
const std::string FILE_SORT_ORDER_BY_KEY
std::string append_file_path(const std::optional< std::string > &base, const std::optional< std::string > &subdirectory)
std::optional< std::string > getOption(const std::string_view &key) const
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:771
const std::set< std::string_view > & getSupportedTableOptions() const override
void validateServerOptions(const ForeignServer *foreign_server) const override
const ForeignServer * foreign_server
Definition: ForeignTable.h:56
#define CHECK(condition)
Definition: Logger.h:223
static const std::set< std::string_view > supported_table_options_
static std::string getFullFilePath(const ForeignTable *foreign_table)
Returns the path to the source file/dir of the table. Depending on options this may result from a con...
static void validateFilePath(const ForeignTable *foreign_table)