OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
AbstractFileStorageDataWrapper.cpp
Go to the documentation of this file.
2 
4 #include "Catalog/ForeignTable.h"
6 #include "Shared/misc.h"
7 #include "Utils/DdlUtils.h"
8 
9 extern bool g_enable_s3_fsi;
10 
11 namespace foreign_storage {
13 
15  const ForeignServer* foreign_server) const {
16  const auto& options = foreign_server->options;
17  for (const auto& entry : options) {
18  if (!shared::contains(supported_server_options_, entry.first)) {
19  throw std::runtime_error{"Invalid foreign server option \"" + entry.first +
20  "\". Option must be one of the following: " +
21  join(supported_server_options_, ", ") + "."};
22  }
23  }
24 
25  if (options.find(STORAGE_TYPE_KEY) == options.end()) {
26  throw std::runtime_error{"Foreign server options must contain \"" + STORAGE_TYPE_KEY +
27  "\"."};
28  }
29  const auto& storage_type = options.find(STORAGE_TYPE_KEY)->second;
30  if (!shared::contains(supported_storage_types, storage_type)) {
31  throw std::runtime_error{"Invalid \"" + STORAGE_TYPE_KEY +
32  "\" option value. Value must be one of the following: " +
33  join(supported_storage_types, ", ") + "."};
34  }
35 
36  if (!g_enable_s3_fsi && storage_type == S3_STORAGE_TYPE) {
37  throw std::runtime_error{
38  "Foreign server storage type value of \"" + std::string{S3_STORAGE_TYPE} +
39  "\" is not allowed because FSI S3 support is currently disabled."};
40  }
41 }
42 
44  const ForeignTable* foreign_table) const {
45  validateFilePathOptionKey(foreign_table);
46  validateFilePath(foreign_table);
48  foreign_table->getOption(FILE_SORT_REGEX_KEY));
49 }
50 
51 const std::set<std::string_view>&
54 }
55 
57  const UserMapping* user_mapping,
58  const ForeignServer* foreign_server) const {
59  throw std::runtime_error{"User mapping for the \"" + foreign_server->data_wrapper_type +
60  "\" data wrapper can only be created for AWS S3 backed "
61  "foreign servers. AWS S3 support is currently disabled."};
62 }
63 
64 const std::set<std::string_view>&
67 }
68 
70  auto& server_options = foreign_table->foreign_server->options;
71  if (server_options.find(STORAGE_TYPE_KEY)->second == LOCAL_FILE_STORAGE_TYPE) {
74  }
75 }
76 
77 namespace {
78 std::string append_file_path(const std::optional<std::string>& base,
79  const std::optional<std::string>& subdirectory) {
80  const std::string separator{boost::filesystem::path::preferred_separator};
81  return std::regex_replace(
82  (base ? *base + separator : "") + (subdirectory ? *subdirectory : ""),
83  std::regex{separator + "{2,}"},
84  separator);
85 }
86 } // namespace
87 
93  const ForeignTable* foreign_table) {
94  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
95  std::optional<std::string> base_path{};
96  auto foreign_server = foreign_table->foreign_server;
97  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
98  CHECK(storage_type);
99 
100  const std::string separator{boost::filesystem::path::preferred_separator};
101  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
102  base_path = foreign_server->getOption(BASE_PATH_KEY);
103  }
104 
105  // If both base_path and file_path are present, then concatenate. Otherwise we are just
106  // taking the one as the path. One of the two must exist, or we have failed validation.
107  CHECK(file_path || base_path);
108  return append_file_path(base_path, file_path);
109 }
110 
111 namespace {
112 void throw_file_path_error(const std::string_view& missing_path,
113  const std::string& table_name,
114  const std::string_view& file_path_key) {
115  std::stringstream ss;
116  ss << "No file_path found for Foreign Table \"" << table_name
117  << "\". Table must have either set a \"" << file_path_key << "\" option, or its "
118  << "parent server must have set a \"" << missing_path << "\" option.";
119  throw std::runtime_error(ss.str());
120 }
121 } // namespace
122 
123 // A valid path is a concatenation of the file_path and the base_path (for local storage).
124 // One of the two must be present.
126  const ForeignTable* foreign_table) {
127  auto file_path = foreign_table->getOption(FILE_PATH_KEY);
128  auto foreign_server = foreign_table->foreign_server;
129  auto storage_type = foreign_server->getOption(STORAGE_TYPE_KEY);
130  CHECK(storage_type) << "No storage type found in parent server. Server \""
131  << foreign_server->name << "\" is not valid.";
132  if (!file_path) {
133  if (*storage_type == LOCAL_FILE_STORAGE_TYPE) {
134  if (!foreign_server->getOption(BASE_PATH_KEY)) {
136  }
137  } else {
138  UNREACHABLE() << "Unknown foreign storage type.";
139  }
140  }
141 }
142 
143 const std::set<std::string_view> AbstractFileStorageDataWrapper::supported_table_options_{
144  FILE_PATH_KEY,
145  REGEX_PATH_FILTER_KEY,
148 
149 const std::set<std::string_view>
151  BASE_PATH_KEY};
152 
153 const std::set<std::string_view>
155 } // namespace foreign_storage
void throw_file_path_error(const std::string_view &missing_path, const std::string &table_name, const std::string_view &file_path_key)
const std::set< std::string_view > & getSupportedUserMappingOptions() const override
bool contains(const T &container, const U &element)
Definition: misc.h:188
static void validateFilePathOptionKey(const ForeignTable *foreign_table)
static const std::array< std::string, 1 > supported_storage_types
const std::string FILE_SORT_REGEX_KEY
std::string tableName
void validate_sort_options(const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex)
std::string join(T const &container, std::string const &delim)
#define UNREACHABLE()
Definition: Logger.h:253
void validateUserMappingOptions(const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
static const std::set< std::string_view > supported_server_options_
void validateTableOptions(const ForeignTable *foreign_table) const override
static const std::set< std::string_view > supported_user_mapping_options_
bool g_enable_s3_fsi
Definition: Catalog.cpp:94
const std::string FILE_SORT_ORDER_BY_KEY
std::string append_file_path(const std::optional< std::string > &base, const std::optional< std::string > &subdirectory)
std::optional< std::string > getOption(const std::string_view &key) const
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:770
const std::set< std::string_view > & getSupportedTableOptions() const override
void validateServerOptions(const ForeignServer *foreign_server) const override
const ForeignServer * foreign_server
Definition: ForeignTable.h:54
#define CHECK(condition)
Definition: Logger.h:209
static const std::set< std::string_view > supported_table_options_
static std::string getFullFilePath(const ForeignTable *foreign_table)
Returns the path to the source file/dir of the table. Depending on options this may result from a con...
static void validateFilePath(const ForeignTable *foreign_table)