OmniSciDB  a7179b2938
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ForeignTable.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "ForeignTable.h"
18 #include <boost/algorithm/string/predicate.hpp>
19 #include <regex>
22 #include "Shared/DateTimeParser.h"
23 
24 namespace foreign_storage {
30 
31 std::vector<std::string_view> ForeignTable::getSupportedDataWrapperOptions() const {
34  } else if (foreign_server->data_wrapper_type ==
37  }
38  return {};
39 }
40 
41 void ForeignTable::validateOptions() const {
44 }
45 
46 bool ForeignTable::isAppendMode() const {
47  auto update_mode = options.find(REFRESH_UPDATE_TYPE_KEY);
48  return (update_mode != options.end() &&
49  update_mode->second == APPEND_REFRESH_UPDATE_TYPE);
50 }
51 
52 std::string ForeignTable::getFilePath() const {
53  auto& server_options = foreign_server->options;
54  auto file_path_entry = options.find("FILE_PATH");
55  std::string file_path{};
56  if (file_path_entry != options.end()) {
57  file_path = file_path_entry->second;
58  }
59  std::string base_path{};
60  if (server_options.find(ForeignServer::STORAGE_TYPE_KEY)->second ==
62  auto base_path_entry = server_options.find(ForeignServer::BASE_PATH_KEY);
63  if (base_path_entry == server_options.end()) {
64  throw std::runtime_error{"No base path found in foreign server options."};
65  }
66  base_path = base_path_entry->second;
67  const std::string separator{boost::filesystem::path::preferred_separator};
68  return std::regex_replace(
69  base_path + separator + file_path, std::regex{separator + "{2,}"}, separator);
70  } else {
71  // Just return the file path as a prefix
72  return file_path;
73  }
74 }
75 
76 void ForeignTable::initializeOptions(const rapidjson::Value& options) {
77  // Create the options map first because the json version is not guaranteed to be
78  // upper-case, which we need to compare reliably with alterable_options.
79  auto options_map = create_options_map(options);
80  validateSupportedOptions(options_map);
81  populateOptionsMap(std::move(options_map));
83 }
84 
85 // This function can't be static because it needs to know the data wrapper type.
86 void ForeignTable::validateSupportedOptions(const OptionsMap& options_map) const {
87  const auto data_wrapper_options = getSupportedDataWrapperOptions();
88  for (const auto& [key, value] : options_map) {
89  if (!contains(supported_options, key) && !contains(data_wrapper_options, key)) {
90  throw std::runtime_error{"Invalid foreign table option \"" + key + "\"."};
91  }
92  }
93 }
94 
96  auto update_type_entry =
98  CHECK(update_type_entry != options.end());
99  auto update_type_value = update_type_entry->second;
100  if (update_type_value != ALL_REFRESH_UPDATE_TYPE &&
101  update_type_value != APPEND_REFRESH_UPDATE_TYPE) {
102  std::string error_message =
103  "Invalid value \"" + update_type_value + "\" for " + REFRESH_UPDATE_TYPE_KEY +
104  " option." + " Value must be \"" + std::string{APPEND_REFRESH_UPDATE_TYPE} +
105  "\" or \"" + std::string{ALL_REFRESH_UPDATE_TYPE} + "\".";
106  throw std::runtime_error{error_message};
107  }
108 
109  auto refresh_timing_entry =
111  CHECK(refresh_timing_entry != options.end());
112  if (auto refresh_timing_value = refresh_timing_entry->second;
113  refresh_timing_value == SCHEDULE_REFRESH_TIMING_TYPE) {
114  auto start_date_entry = options.find(REFRESH_START_DATE_TIME_KEY);
115  if (start_date_entry == options.end()) {
116  throw std::runtime_error{std::string{REFRESH_START_DATE_TIME_KEY} +
117  " option must be provided for scheduled refreshes."};
118  }
119  auto start_date_time = dateTimeParse<kTIMESTAMP>(start_date_entry->second, 0);
120  int64_t current_time = std::chrono::duration_cast<std::chrono::seconds>(
121  std::chrono::system_clock::now().time_since_epoch())
122  .count();
123  if (start_date_time < current_time) {
124  throw std::runtime_error{std::string{REFRESH_START_DATE_TIME_KEY} +
125  " cannot be a past date time."};
126  }
127 
128  auto interval_entry = options.find(REFRESH_INTERVAL_KEY);
129  if (interval_entry != options.end()) {
130  boost::regex interval_regex{"^\\d{1,}[SHD]$",
131  boost::regex::extended | boost::regex::icase};
132  if (!boost::regex_match(interval_entry->second, interval_regex)) {
133  throw std::runtime_error{"Invalid value provided for the " +
134  std::string{REFRESH_INTERVAL_KEY} + " option."};
135  }
136  }
137  } else if (refresh_timing_value != MANUAL_REFRESH_TIMING_TYPE) {
138  throw std::runtime_error{"Invalid value provided for the " +
139  std::string{REFRESH_TIMING_TYPE_KEY} +
140  " option. Value must be \"" + MANUAL_REFRESH_TIMING_TYPE +
141  "\" or \"" + SCHEDULE_REFRESH_TIMING_TYPE + "\"."};
142  }
143 }
144 
146  const auto wrapper_type = foreign_server->data_wrapper_type;
147  if (wrapper_type == foreign_storage::DataWrapperType::CSV) {
149  } else if (wrapper_type == foreign_storage::DataWrapperType::PARQUET) {
151  } else {
152  UNREACHABLE() << "Unknown data wrapper type";
153  }
154 }
155 
156 OptionsMap ForeignTable::create_options_map(const rapidjson::Value& json_options) {
157  OptionsMap options_map;
158  CHECK(json_options.IsObject());
159  for (const auto& member : json_options.GetObject()) {
160  auto key = to_upper(member.name.GetString());
161  if (std::find(upper_case_options.begin(), upper_case_options.end(), key) !=
162  upper_case_options.end()) {
163  options_map[key] = to_upper(member.value.GetString());
164  } else {
165  options_map[key] = member.value.GetString();
166  }
167  }
168  return options_map;
169 }
170 
171 void ForeignTable::validate_alter_options(const OptionsMap& options_map) {
172  for (const auto& [key, value] : options_map) {
173  if (!contains(alterable_options, key)) {
174  throw std::runtime_error{std::string("Altering foreign table option \"") + key +
175  "\" is not currently supported."};
176  }
177  }
178 }
179 
180 } // namespace foreign_storage
void validateDataWrapperOptions() const
static const std::set< const char * > supported_options
Definition: ForeignTable.h:65
static std::vector< std::string_view > getSupportedOptions()
static std::vector< std::string_view > getSupportedOptions()
static void validateOptions(const ForeignTable *foreign_table)
int64_t dateTimeParse< kTIMESTAMP >(std::string_view str, unsigned const dim)
#define UNREACHABLE()
Definition: Logger.h:241
bool contains(const T &set, const std::string_view element)
Definition: ForeignTable.h:26
static constexpr const char * MANUAL_REFRESH_TIMING_TYPE
Definition: ForeignTable.h:60
static void validate_alter_options(const OptionsMap &options_map)
static constexpr std::string_view STORAGE_TYPE_KEY
Definition: ForeignServer.h:43
static constexpr const char * REFRESH_START_DATE_TIME_KEY
Definition: ForeignTable.h:53
static constexpr const char * REFRESH_UPDATE_TYPE_KEY
Definition: ForeignTable.h:55
void validateRefreshOptions() const
void populateOptionsMap(OptionsMap &&options_map, bool clear=false)
static constexpr const char * REFRESH_INTERVAL_KEY
Definition: ForeignTable.h:54
std::vector< std::string_view > getSupportedDataWrapperOptions() const
std::string to_upper(const std::string &str)
std::string getFilePath() const
static const std::set< const char * > alterable_options
Definition: ForeignTable.h:79
static constexpr const char * ALL_REFRESH_UPDATE_TYPE
Definition: ForeignTable.h:57
static constexpr std::string_view LOCAL_FILE_STORAGE_TYPE
Definition: ForeignServer.h:45
static OptionsMap create_options_map(const rapidjson::Value &json_options)
void initializeOptions(const rapidjson::Value &options)
static constexpr const char * APPEND_REFRESH_UPDATE_TYPE
Definition: ForeignTable.h:58
const ForeignServer * foreign_server
Definition: ForeignTable.h:63
static constexpr const char * REFRESH_TIMING_TYPE_KEY
Definition: ForeignTable.h:52
static constexpr std::string_view BASE_PATH_KEY
Definition: ForeignServer.h:44
#define CHECK(condition)
Definition: Logger.h:197
static constexpr char const * CSV
Definition: ForeignServer.h:35
std::map< std::string, std::string, std::less<>> OptionsMap
static void validateOptions(const ForeignTable *foreign_table)
void validateSupportedOptions(const OptionsMap &options_map) const
static constexpr char const * PARQUET
Definition: ForeignServer.h:36
static constexpr const char * SCHEDULE_REFRESH_TIMING_TYPE
Definition: ForeignTable.h:59
static const std::set< const char * > upper_case_options
Definition: ForeignTable.h:72