OmniSciDB  dfae7c3b14
DdlUtils.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "DdlUtils.h"
18 
19 #include <unordered_set>
20 
21 #include <boost/algorithm/string.hpp>
22 #include <boost/filesystem.hpp>
23 #include <boost/program_options.hpp>
24 
25 #include "rapidjson/document.h"
26 
30 #include "Shared/misc.h"
31 
33 
34 namespace ddl_utils {
35 SqlType::SqlType(SQLTypes type, int param1, int param2, bool is_array, int array_size)
36  : type(type)
37  , param1(param1)
38  , param2(param2)
39  , is_array(is_array)
40  , array_size(array_size) {}
41 
43  return type;
44 }
45 
46 int SqlType::get_param1() const {
47  return param1;
48 }
49 
50 void SqlType::set_param1(int param) {
51  param1 = param;
52 }
53 
54 int SqlType::get_param2() const {
55  return param2;
56 }
57 
58 bool SqlType::get_is_array() const {
59  return is_array;
60 }
61 
62 void SqlType::set_is_array(bool a) {
63  is_array = a;
64 }
65 
67  return array_size;
68 }
69 
71  array_size = s;
72 }
73 
74 std::string SqlType::to_string() const {
75  std::string str;
76  switch (type) {
77  case kBOOLEAN:
78  str = "BOOLEAN";
79  break;
80  case kCHAR:
81  str = "CHAR(" + boost::lexical_cast<std::string>(param1) + ")";
82  break;
83  case kVARCHAR:
84  str = "VARCHAR(" + boost::lexical_cast<std::string>(param1) + ")";
85  break;
86  case kTEXT:
87  str = "TEXT";
88  break;
89  case kNUMERIC:
90  str = "NUMERIC(" + boost::lexical_cast<std::string>(param1);
91  if (param2 > 0) {
92  str += ", " + boost::lexical_cast<std::string>(param2);
93  }
94  str += ")";
95  break;
96  case kDECIMAL:
97  str = "DECIMAL(" + boost::lexical_cast<std::string>(param1);
98  if (param2 > 0) {
99  str += ", " + boost::lexical_cast<std::string>(param2);
100  }
101  str += ")";
102  break;
103  case kBIGINT:
104  str = "BIGINT";
105  break;
106  case kINT:
107  str = "INT";
108  break;
109  case kTINYINT:
110  str = "TINYINT";
111  break;
112  case kSMALLINT:
113  str = "SMALLINT";
114  break;
115  case kFLOAT:
116  str = "FLOAT";
117  break;
118  case kDOUBLE:
119  str = "DOUBLE";
120  break;
121  case kTIME:
122  str = "TIME";
123  if (param1 < 6) {
124  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
125  }
126  break;
127  case kTIMESTAMP:
128  str = "TIMESTAMP";
129  if (param1 <= 9) {
130  str += "(" + boost::lexical_cast<std::string>(param1) + ")";
131  }
132  break;
133  case kDATE:
134  str = "DATE";
135  break;
136  default:
137  assert(false);
138  break;
139  }
140  if (is_array) {
141  str += "[";
142  if (array_size > 0) {
143  str += boost::lexical_cast<std::string>(array_size);
144  }
145  str += "]";
146  }
147  return str;
148 }
149 
151  switch (type) {
152  case kCHAR:
153  case kVARCHAR:
154  if (param1 <= 0) {
155  throw std::runtime_error("CHAR and VARCHAR must have a positive dimension.");
156  }
157  break;
158  case kDECIMAL:
159  case kNUMERIC:
160  if (param1 <= 0) {
161  throw std::runtime_error("DECIMAL and NUMERIC must have a positive precision.");
162  } else if (param1 > 19) {
163  throw std::runtime_error(
164  "DECIMAL and NUMERIC precision cannot be larger than 19.");
165  } else if (param1 <= param2) {
166  throw std::runtime_error(
167  "DECIMAL and NUMERIC must have precision larger than scale.");
168  }
169  break;
170  case kTIMESTAMP:
171  if (param1 == -1) {
172  param1 = 0; // set default to 0
173  } else if (param1 != 0 && param1 != 3 && param1 != 6 &&
174  param1 != 9) { // support ms, us, ns
175  throw std::runtime_error(
176  "Only TIMESTAMP(n) where n = (0,3,6,9) are supported now.");
177  }
178  break;
179  case kTIME:
180  if (param1 == -1) {
181  param1 = 0; // default precision is 0
182  }
183  if (param1 > 0) { // @TODO(wei) support sub-second precision later.
184  throw std::runtime_error("Only TIME(0) is supported now.");
185  }
186  break;
187  case kPOINT:
188  case kLINESTRING:
189  case kPOLYGON:
190  case kMULTIPOLYGON:
191  // Storing SRID in param1
192  break;
193  default:
194  param1 = 0;
195  break;
196  }
197 }
198 
199 Encoding::Encoding(std::string* encoding_name, int encoding_param)
200  : encoding_name(encoding_name), encoding_param(encoding_param) {}
201 
202 const std::string* Encoding::get_encoding_name() const {
203  return encoding_name.get();
204 }
205 
207  return encoding_param;
208 }
209 
211  // Change default TEXT column behaviour to be DICT encoded
212  if (cd.columnType.is_string() || cd.columnType.is_string_array()) {
213  // default to 32-bits
215  cd.columnType.set_comp_param(32);
216  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 4) {
218  cd.columnType.set_comp_param(16);
219  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 9) {
221  cd.columnType.set_comp_param(32);
222  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
223  throw std::runtime_error(cd.columnName + ": Precision too high, max 18.");
224  } else if (cd.columnType.is_geometry() && cd.columnType.get_output_srid() == 4326) {
225  // default to GEOINT 32-bits
227  cd.columnType.set_comp_param(32);
229  // Days encoding for DATE
232  } else {
235  }
236 }
237 
239  int encoding_size,
240  const SqlType* column_type) {
241  if (!cd.columnType.is_integer() && !cd.columnType.is_time() &&
242  !cd.columnType.is_decimal()) {
243  throw std::runtime_error(
244  cd.columnName +
245  ": Fixed encoding is only supported for integer or time columns.");
246  }
247 
248  auto type = cd.columnType.get_type();
249  // fixed-bits encoding
250  if (type == kARRAY) {
251  type = cd.columnType.get_subtype();
252  }
253  switch (type) {
254  case kSMALLINT:
255  if (encoding_size != 8) {
256  throw std::runtime_error(
257  cd.columnName +
258  ": Compression parameter for Fixed encoding on SMALLINT must be 8.");
259  }
260  break;
261  case kINT:
262  if (encoding_size != 8 && encoding_size != 16) {
263  throw std::runtime_error(
264  cd.columnName +
265  ": Compression parameter for Fixed encoding on INTEGER must be 8 or 16.");
266  }
267  break;
268  case kBIGINT:
269  if (encoding_size != 8 && encoding_size != 16 && encoding_size != 32) {
270  throw std::runtime_error(cd.columnName +
271  ": Compression parameter for Fixed encoding on "
272  "BIGINT must be 8 or 16 or 32.");
273  }
274  break;
275  case kTIMESTAMP:
276  case kTIME:
277  if (encoding_size != 32) {
278  throw std::runtime_error(cd.columnName +
279  ": Compression parameter for Fixed encoding on "
280  "TIME or TIMESTAMP must be 32.");
281  } else if (cd.columnType.is_high_precision_timestamp()) {
282  throw std::runtime_error("Fixed encoding is not supported for TIMESTAMP(3|6|9).");
283  }
284  break;
285  case kDECIMAL:
286  case kNUMERIC:
287  if (encoding_size != 32 && encoding_size != 16) {
288  throw std::runtime_error(cd.columnName +
289  ": Compression parameter for Fixed encoding on "
290  "DECIMAL must be 16 or 32.");
291  }
292 
293  if (encoding_size == 32 && cd.columnType.get_precision() > 9) {
294  throw std::runtime_error(cd.columnName +
295  ": Precision too high for Fixed(32) encoding, max 9.");
296  }
297 
298  if (encoding_size == 16 && cd.columnType.get_precision() > 4) {
299  throw std::runtime_error(cd.columnName +
300  ": Precision too high for Fixed(16) encoding, max 4.");
301  }
302  break;
303  case kDATE:
304  if (encoding_size != 32 && encoding_size != 16) {
305  throw std::runtime_error(cd.columnName +
306  ": Compression parameter for Fixed encoding on "
307  "DATE must be 16 or 32.");
308  }
309  break;
310  default:
311  throw std::runtime_error(cd.columnName + ": Cannot apply FIXED encoding to " +
312  column_type->to_string());
313  }
314  if (type == kDATE) {
316  cd.columnType.set_comp_param(16);
317  } else {
319  cd.columnType.set_comp_param(encoding_size);
320  }
321 }
322 
324  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
325  throw std::runtime_error(
326  cd.columnName +
327  ": Dictionary encoding is only supported on string or string array columns.");
328  }
329  int comp_param;
330  if (encoding_size == 0) {
331  comp_param = 32; // default to 32-bits
332  } else {
333  comp_param = encoding_size;
334  }
335  if (cd.columnType.is_string_array() && comp_param != 32) {
336  throw std::runtime_error(cd.columnName +
337  ": Compression parameter for string arrays must be 32");
338  }
339  if (comp_param != 8 && comp_param != 16 && comp_param != 32) {
340  throw std::runtime_error(
341  cd.columnName +
342  ": Compression parameter for Dictionary encoding must be 8 or 16 or 32.");
343  }
344  // dictionary encoding
346  cd.columnType.set_comp_param(comp_param);
347 }
348 
350  if (!cd.columnType.is_string() && !cd.columnType.is_string_array() &&
351  !cd.columnType.is_geometry()) {
352  throw std::runtime_error(
353  cd.columnName +
354  ": None encoding is only supported on string, string array, or geo columns.");
355  }
358 }
359 
361  // sparse column encoding with mostly NULL values
362  if (cd.columnType.get_notnull()) {
363  throw std::runtime_error(cd.columnName +
364  ": Cannot do sparse column encoding on a NOT NULL column.");
365  }
366  if (encoding_size == 0 || encoding_size % 8 != 0 || encoding_size > 48) {
367  throw std::runtime_error(
368  cd.columnName +
369  "Must specify number of bits as 8, 16, 24, 32 or 48 as the parameter to "
370  "sparse-column encoding.");
371  }
373  cd.columnType.set_comp_param(encoding_size);
374  // throw std::runtime_error("SPARSE encoding not supported yet.");
375 }
376 
378  if (!cd.columnType.is_geometry() || cd.columnType.get_output_srid() != 4326) {
379  throw std::runtime_error(
380  cd.columnName + ": COMPRESSED encoding is only supported on WGS84 geo columns.");
381  }
382  int comp_param;
383  if (encoding_size == 0) {
384  comp_param = 32; // default to 32-bits
385  } else {
386  comp_param = encoding_size;
387  }
388  if (comp_param != 32) {
389  throw std::runtime_error(cd.columnName +
390  ": only 32-bit COMPRESSED geo encoding is supported");
391  }
392  // encoding longitude/latitude as integers
394  cd.columnType.set_comp_param(comp_param);
395 }
396 
397 void validate_and_set_date_encoding(ColumnDescriptor& cd, int encoding_size) {
398  // days encoding for dates
399  if (cd.columnType.get_type() != kDATE) {
400  throw std::runtime_error(cd.columnName +
401  ": Days encoding is only supported for DATE columns.");
402  }
403  if (encoding_size != 32 && encoding_size != 16) {
404  throw std::runtime_error(cd.columnName +
405  ": Compression parameter for Days encoding on "
406  "DATE must be 16 or 32.");
407  }
409  cd.columnType.set_comp_param((encoding_size == 16) ? 16 : 0);
410 }
411 
413  const Encoding* encoding,
414  const SqlType* column_type) {
415  if (encoding == nullptr) {
417  } else {
418  const std::string& comp = *encoding->get_encoding_name();
419  if (boost::iequals(comp, "fixed")) {
420  validate_and_set_fixed_encoding(cd, encoding->get_encoding_param(), column_type);
421  } else if (boost::iequals(comp, "rl")) {
422  // run length encoding
425  // throw std::runtime_error("RL(Run Length) encoding not supported yet.");
426  } else if (boost::iequals(comp, "diff")) {
427  // differential encoding
430  // throw std::runtime_error("DIFF(differential) encoding not supported yet.");
431  } else if (boost::iequals(comp, "dict")) {
433  } else if (boost::iequals(comp, "NONE")) {
435  } else if (boost::iequals(comp, "sparse")) {
437  } else if (boost::iequals(comp, "compressed")) {
439  } else if (boost::iequals(comp, "days")) {
441  } else {
442  throw std::runtime_error(cd.columnName + ": Invalid column compression scheme " +
443  comp);
444  }
445  }
446 }
447 
449  column_type->check_type();
450 
451  if (column_type->get_is_array()) {
453  cd.columnType.set_subtype(column_type->get_type());
454  } else {
455  cd.columnType.set_type(column_type->get_type());
456  }
457  if (IS_GEO(column_type->get_type())) {
458  cd.columnType.set_subtype(static_cast<SQLTypes>(column_type->get_param1()));
459  cd.columnType.set_input_srid(column_type->get_param2());
460  cd.columnType.set_output_srid(column_type->get_param2());
461  } else {
462  cd.columnType.set_dimension(column_type->get_param1());
463  cd.columnType.set_scale(column_type->get_param2());
464  }
465 }
466 
467 void validate_and_set_array_size(ColumnDescriptor& cd, const SqlType* column_type) {
468  if (cd.columnType.is_string_array() &&
470  throw std::runtime_error(
471  cd.columnName +
472  ": Array of strings must be dictionary encoded. Specify ENCODING DICT");
473  }
474 
475  if (column_type->get_is_array()) {
476  int s = -1;
477  auto array_size = column_type->get_array_size();
478  if (array_size > 0) {
479  auto sti = cd.columnType.get_elem_type();
480  s = array_size * sti.get_size();
481  if (s <= 0) {
482  throw std::runtime_error(cd.columnName + ": Unexpected fixed length array size");
483  }
484  }
485  cd.columnType.set_size(s);
486 
487  } else {
489  }
490 }
491 
492 void set_column_descriptor(const std::string& column_name,
493  ColumnDescriptor& cd,
494  SqlType* column_type,
495  const bool not_null,
496  const Encoding* encoding) {
497  cd.columnName = column_name;
498  validate_and_set_type(cd, column_type);
499  cd.columnType.set_notnull(not_null);
500  validate_and_set_encoding(cd, encoding, column_type);
501  validate_and_set_array_size(cd, column_type);
502  cd.isSystemCol = false;
503  cd.isVirtualCol = false;
504 }
505 
506 void set_default_table_attributes(const std::string& table_name,
507  TableDescriptor& td,
508  const int32_t column_count) {
509  td.tableName = table_name;
510  td.nColumns = column_count;
511  td.isView = false;
512  td.fragmenter = nullptr;
518 }
519 
520 void validate_non_duplicate_column(const std::string& column_name,
521  std::unordered_set<std::string>& upper_column_names) {
522  const auto upper_column_name = boost::to_upper_copy<std::string>(column_name);
523  const auto insert_it = upper_column_names.insert(upper_column_name);
524  if (!insert_it.second) {
525  throw std::runtime_error("Column '" + column_name + "' defined more than once");
526  }
527 }
528 
529 void validate_non_reserved_keyword(const std::string& column_name) {
530  const auto upper_column_name = boost::to_upper_copy<std::string>(column_name);
531  if (reserved_keywords.find(upper_column_name) != reserved_keywords.end()) {
532  throw std::runtime_error("Cannot create column with reserved keyword '" +
533  column_name + "'");
534  }
535 }
536 
538  const TableType expected_table_type) {
539  if (td->isView) {
540  if (expected_table_type != TableType::VIEW) {
541  throw std::runtime_error(td->tableName + " is a view. Use DROP VIEW.");
542  }
543  } else if (td->storageType == StorageType::FOREIGN_TABLE) {
544  if (expected_table_type != TableType::FOREIGN_TABLE) {
545  throw std::runtime_error(td->tableName +
546  " is a foreign table. Use DROP FOREIGN TABLE.");
547  }
548  } else if (expected_table_type != TableType::TABLE) {
549  throw std::runtime_error(td->tableName + " is a table. Use DROP TABLE.");
550  }
551 }
552 
553 std::string table_type_enum_to_string(const TableType table_type) {
554  if (table_type == ddl_utils::TableType::TABLE) {
555  return "Table";
556  }
557  if (table_type == ddl_utils::TableType::FOREIGN_TABLE) {
558  return "ForeignTable";
559  }
560  if (table_type == ddl_utils::TableType::VIEW) {
561  return "View";
562  }
563  throw std::runtime_error{"Unexpected table type"};
564 }
565 
566 std::string get_malformed_config_error_message(const std::string& config_key) {
567  return "Configuration value for \"" + config_key +
568  "\" is malformed. Value should be a list of paths with format: [ "
569  "\"root-path-1\", \"root-path-2\", ... ]";
570 }
571 
572 void validate_expanded_file_path(const std::string& file_path,
573  const std::vector<std::string>& whitelisted_root_paths) {
574  boost::filesystem::path canonical_file_path = boost::filesystem::canonical(file_path);
575  for (const auto& root_path : whitelisted_root_paths) {
576  if (boost::istarts_with(canonical_file_path.string(), root_path)) {
577  return;
578  }
579  }
580  throw std::runtime_error{"File or directory path \"" + file_path +
581  "\" is not whitelisted."};
582 }
583 
584 std::vector<std::string> get_expanded_file_paths(
585  const std::string& file_path,
586  const DataTransferType data_transfer_type) {
587  std::vector<std::string> file_paths;
588  if (data_transfer_type == DataTransferType::IMPORT) {
589  file_paths = omnisci::glob(file_path);
590  if (file_paths.size() == 0) {
591  throw std::runtime_error{"File or directory \"" + file_path + "\" does not exist."};
592  }
593  } else {
594  std::string path;
595  if (!boost::filesystem::exists(file_path)) {
596  // For exports, it is possible to provide a path to a new (nonexistent) file. In
597  // this case, validate using the parent path.
598  path = boost::filesystem::path(file_path).parent_path().string();
599  if (!boost::filesystem::exists(path)) {
600  throw std::runtime_error{"File or directory \"" + file_path +
601  "\" does not exist."};
602  }
603  } else {
604  path = file_path;
605  }
606  file_paths = {path};
607  }
608  return file_paths;
609 }
610 
611 void validate_allowed_file_path(const std::string& file_path,
612  const DataTransferType data_transfer_type,
613  const bool allow_wildcards) {
614  // Reject any punctuation characters except for a few safe ones.
615  // Some punctuation characters present a security risk when passed
616  // to subprocesses. Don't change this without a security review.
617  static const std::string safe_punctuation{"./_+-=:~"};
618  for (const auto& ch : file_path) {
619  if (std::ispunct(ch) && safe_punctuation.find(ch) == std::string::npos &&
620  !(allow_wildcards && ch == '*')) {
621  throw std::runtime_error(std::string("Punctuation \"") + ch +
622  "\" is not allowed in file path: " + file_path);
623  }
624  }
625 
626  // Enforce our whitelist and blacklist for file paths.
627  const auto& expanded_file_paths =
628  get_expanded_file_paths(file_path, data_transfer_type);
629  for (const auto& path : expanded_file_paths) {
631  throw std::runtime_error{"Access to file or directory path \"" + file_path +
632  "\" is not allowed."};
633  }
634  }
635  FilePathWhitelist::validateWhitelistedFilePath(expanded_file_paths, data_transfer_type);
636 }
637 
638 void set_whitelisted_paths(const std::string& config_key,
639  const std::string& config_value,
640  std::vector<std::string>& whitelisted_paths) {
641  CHECK(whitelisted_paths.empty());
642  rapidjson::Document whitelisted_root_paths;
643  whitelisted_root_paths.Parse(config_value);
644  if (!whitelisted_root_paths.IsArray()) {
645  throw std::runtime_error{get_malformed_config_error_message(config_key)};
646  }
647  for (const auto& root_path : whitelisted_root_paths.GetArray()) {
648  if (!root_path.IsString()) {
649  throw std::runtime_error{get_malformed_config_error_message(config_key)};
650  }
651  if (!boost::filesystem::exists(root_path.GetString())) {
652  throw std::runtime_error{"Whitelisted root path \"" +
653  std::string{root_path.GetString()} + "\" does not exist."};
654  }
655  whitelisted_paths.emplace_back(
656  boost::filesystem::canonical(root_path.GetString()).string());
657  }
658  LOG(INFO) << config_key << " " << shared::printContainer(whitelisted_paths);
659 }
660 
661 void FilePathWhitelist::initializeFromConfigFile(const std::string& server_config_path) {
662  if (server_config_path.empty()) {
663  return;
664  }
665 
666  if (!boost::filesystem::exists(server_config_path)) {
667  throw std::runtime_error{"Configuration file at \"" + server_config_path +
668  "\" does not exist."};
669  }
670 
671  std::string import_config_key{"allowed-import-paths"};
672  std::string export_config_key{"allowed-export-paths"};
673  std::string import_config_value, export_config_value;
674 
675  namespace po = boost::program_options;
676  po::options_description desc{};
677  desc.add_options()(import_config_key.c_str(),
678  po::value<std::string>(&import_config_value));
679  desc.add_options()(export_config_key.c_str(),
680  po::value<std::string>(&export_config_value));
681  po::variables_map vm;
682  po::store(po::parse_config_file<char>(server_config_path.c_str(), desc, true), vm);
683  po::notify(vm);
684  if (vm.count(import_config_key)) {
686  import_config_key, import_config_value, whitelisted_import_paths_);
687  }
688  if (vm.count(export_config_key)) {
690  export_config_key, export_config_value, whitelisted_export_paths_);
691  }
692 }
693 
695  const std::vector<std::string>& expanded_file_paths,
696  const DataTransferType data_transfer_type) {
697  // Skip validation if no whitelist configuration is provided.
698  if ((data_transfer_type == DataTransferType::IMPORT &&
699  whitelisted_import_paths_.empty()) ||
700  (data_transfer_type == DataTransferType::EXPORT &&
701  whitelisted_export_paths_.empty())) {
702  return;
703  }
704 
705  for (const auto& path : expanded_file_paths) {
706  if (data_transfer_type == DataTransferType::IMPORT) {
707  validate_expanded_file_path(path, whitelisted_import_paths_);
708  } else if (data_transfer_type == DataTransferType::EXPORT) {
709  validate_expanded_file_path(path, whitelisted_export_paths_);
710  } else {
711  UNREACHABLE();
712  }
713  }
714 }
715 
717  whitelisted_import_paths_.clear();
718  whitelisted_export_paths_.clear();
719 }
720 
721 std::vector<std::string> FilePathWhitelist::whitelisted_import_paths_{};
722 std::vector<std::string> FilePathWhitelist::whitelisted_export_paths_{};
723 
724 void FilePathBlacklist::addToBlacklist(const std::string& path) {
725  CHECK(!path.empty());
726  blacklisted_paths_.emplace_back(path);
727 }
728 
729 bool FilePathBlacklist::isBlacklistedPath(const std::string& path) {
730  const auto canonical_path = boost::filesystem::canonical(path).string();
731  for (const auto& blacklisted_path : blacklisted_paths_) {
732  std::string full_path;
733  try {
734  full_path = boost::filesystem::canonical(blacklisted_path).string();
735  } catch (...) {
742  full_path = boost::filesystem::absolute(blacklisted_path).string();
743  }
744  if (boost::istarts_with(canonical_path, full_path)) {
745  return true;
746  }
747  }
748  return false;
749 }
750 
752  blacklisted_paths_.clear();
753 }
754 
755 std::vector<std::string> FilePathBlacklist::blacklisted_paths_{};
756 } // namespace ddl_utils
virtual int get_array_size() const
Definition: DdlUtils.cpp:66
static std::set< std::string > reserved_keywords
DataTransferType
Definition: DdlUtils.h:80
void set_compression(EncodingType c)
Definition: sqltypes.h:359
void set_size(int s)
Definition: sqltypes.h:357
void validate_and_set_sparse_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:360
bool is_time() const
Definition: sqltypes.h:423
std::vector< std::string > get_expanded_file_paths(const std::string &file_path, const DataTransferType data_transfer_type)
Definition: DdlUtils.cpp:584
static std::vector< std::string > whitelisted_export_paths_
Definition: DdlUtils.h:92
int get_precision() const
Definition: sqltypes.h:262
bool is_string() const
Definition: sqltypes.h:417
Definition: sqltypes.h:51
SQLTypes
Definition: sqltypes.h:40
std::string tableName
SqlType(SQLTypes type, int param1, int param2, bool is_array, int array_size)
Definition: DdlUtils.cpp:35
bool is_integer() const
Definition: sqltypes.h:419
void validate_and_set_array_size(ColumnDescriptor &cd, const SqlType *column_type)
Definition: DdlUtils.cpp:467
virtual void check_type()
Definition: DdlUtils.cpp:150
void validate_and_set_dictionary_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:323
virtual int get_param2() const
Definition: DdlUtils.cpp:54
#define LOG(tag)
Definition: Logger.h:188
std::string storageType
#define DEFAULT_MAX_CHUNK_SIZE
static void initializeFromConfigFile(const std::string &server_config_path)
Definition: DdlUtils.cpp:661
#define UNREACHABLE()
Definition: Logger.h:241
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:350
HOST DEVICE int get_size() const
Definition: sqltypes.h:269
bool is_high_precision_timestamp() const
Definition: sqltypes.h:644
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:267
bool is_decimal() const
Definition: sqltypes.h:420
virtual const std::string * get_encoding_name() const
Definition: DdlUtils.cpp:202
void validate_non_duplicate_column(const std::string &column_name, std::unordered_set< std::string > &upper_column_names)
Definition: DdlUtils.cpp:520
void validate_and_set_none_encoding(ColumnDescriptor &cd)
Definition: DdlUtils.cpp:349
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:266
void set_input_srid(int d)
Definition: sqltypes.h:353
void validate_and_set_encoding(ColumnDescriptor &cd, const Encoding *encoding, const SqlType *column_type)
Definition: DdlUtils.cpp:412
bool g_use_date_in_days_default_encoding
Definition: DdlUtils.cpp:32
#define DEFAULT_MAX_ROWS
static std::vector< std::string > whitelisted_import_paths_
Definition: DdlUtils.h:91
static void validateWhitelistedFilePath(const std::vector< std::string > &expanded_file_paths, const DataTransferType data_transfer_type)
Definition: DdlUtils.cpp:694
void set_fixed_size()
Definition: sqltypes.h:358
HOST DEVICE int get_output_srid() const
Definition: sqltypes.h:265
void set_default_encoding(ColumnDescriptor &cd)
Definition: DdlUtils.cpp:210
std::vector< std::string > glob(const std::string &pattern)
void set_scale(int s)
Definition: sqltypes.h:354
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:260
SQLTypes type
Definition: DdlUtils.h:54
void validate_expanded_file_path(const std::string &file_path, const std::vector< std::string > &whitelisted_root_paths)
Definition: DdlUtils.cpp:572
void validate_non_reserved_keyword(const std::string &column_name)
Definition: DdlUtils.cpp:529
specifies the content in-memory of a row in the column metadata table
void set_default_table_attributes(const std::string &table_name, TableDescriptor &td, const int32_t column_count)
Definition: DdlUtils.cpp:506
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:611
void set_output_srid(int s)
Definition: sqltypes.h:355
#define DEFAULT_PAGE_SIZE
void set_comp_param(int p)
Definition: sqltypes.h:360
void set_column_descriptor(const std::string &column_name, ColumnDescriptor &cd, SqlType *column_type, const bool not_null, const Encoding *encoding)
Definition: DdlUtils.cpp:492
void validate_and_set_compressed_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:377
encoding
Definition: setup.py:9
Definition: sqltypes.h:54
Definition: sqltypes.h:55
bool is_geometry() const
Definition: sqltypes.h:429
bool is_string_array() const
Definition: sqltypes.h:418
void validate_drop_table_type(const TableDescriptor *td, const TableType expected_table_type)
Definition: DdlUtils.cpp:537
std::string get_malformed_config_error_message(const std::string &config_key)
Definition: DdlUtils.cpp:566
void set_dimension(int d)
Definition: sqltypes.h:351
#define DEFAULT_FRAGMENT_ROWS
void validate_and_set_fixed_encoding(ColumnDescriptor &cd, int encoding_size, const SqlType *column_type)
Definition: DdlUtils.cpp:238
std::string table_type_enum_to_string(const TableType table_type)
Definition: DdlUtils.cpp:553
Fragmenter_Namespace::FragmenterType fragType
Encoding(std::string *encoding_name, int encoding_param)
Definition: DdlUtils.cpp:199
void set_whitelisted_paths(const std::string &config_key, const std::string &config_value, std::vector< std::string > &whitelisted_paths)
Definition: DdlUtils.cpp:638
Definition: sqltypes.h:43
virtual void set_param1(int param)
Definition: DdlUtils.cpp:50
static bool isBlacklistedPath(const std::string &path)
Definition: DdlUtils.cpp:729
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:624
void set_notnull(bool n)
Definition: sqltypes.h:356
#define CHECK(condition)
Definition: Logger.h:197
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:259
virtual int get_param1() const
Definition: DdlUtils.cpp:46
virtual bool get_is_array() const
Definition: DdlUtils.cpp:58
void validate_and_set_date_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:397
static void addToBlacklist(const std::string &path)
Definition: DdlUtils.cpp:724
virtual std::string to_string() const
Definition: DdlUtils.cpp:74
Definition: sqltypes.h:47
std::unique_ptr< std::string > encoding_name
Definition: DdlUtils.h:76
SQLTypeInfo columnType
virtual void set_is_array(bool a)
Definition: DdlUtils.cpp:62
PrintContainer< CONTAINER > printContainer(CONTAINER &container)
Definition: misc.h:64
specifies the content in-memory of a row in the table metadata table
virtual SQLTypes get_type() const
Definition: DdlUtils.cpp:42
static constexpr char const * FOREIGN_TABLE
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:174
virtual int get_encoding_param() const
Definition: DdlUtils.cpp:206
virtual void set_array_size(int s)
Definition: DdlUtils.cpp:70
static std::vector< std::string > blacklisted_paths_
Definition: DdlUtils.h:102
void validate_and_set_type(ColumnDescriptor &cd, SqlType *column_type)
Definition: DdlUtils.cpp:448
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:349