OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ddl_utils Namespace Reference

Namespaces

 anonymous_namespace{DdlUtils.cpp}
 

Classes

class  SqlType
 
class  Encoding
 
class  FilePathWhitelist
 
class  FilePathBlacklist
 

Enumerations

enum  DataTransferType { DataTransferType::IMPORT = 1, DataTransferType::EXPORT }
 
enum  TableType { TableType::TABLE = 1, TableType::VIEW, TableType::FOREIGN_TABLE }
 

Functions

void set_default_encoding (ColumnDescriptor &cd)
 
void validate_and_set_fixed_encoding (ColumnDescriptor &cd, int encoding_size, const SqlType *column_type)
 
void validate_and_set_dictionary_encoding (ColumnDescriptor &cd, int encoding_size)
 
void validate_and_set_none_encoding (ColumnDescriptor &cd)
 
void validate_and_set_sparse_encoding (ColumnDescriptor &cd, int encoding_size)
 
void validate_and_set_compressed_encoding (ColumnDescriptor &cd, int encoding_size)
 
void validate_and_set_date_encoding (ColumnDescriptor &cd, int encoding_size)
 
void validate_and_set_encoding (ColumnDescriptor &cd, const Encoding *encoding, const SqlType *column_type)
 
void validate_and_set_type (ColumnDescriptor &cd, SqlType *column_type)
 
void validate_and_set_array_size (ColumnDescriptor &cd, const SqlType *column_type)
 
void validate_and_set_default_value (ColumnDescriptor &cd, const std::string *default_value, bool not_null)
 
void set_column_descriptor (const std::string &column_name, ColumnDescriptor &cd, SqlType *column_type, const bool not_null, const Encoding *encoding, const std::string *default_value)
 
void set_default_table_attributes (const std::string &table_name, TableDescriptor &td, const int32_t column_count)
 
void validate_non_duplicate_column (const std::string &column_name, std::unordered_set< std::string > &upper_column_names)
 
void validate_non_reserved_keyword (const std::string &column_name)
 
void validate_table_type (const TableDescriptor *td, const TableType expected_table_type, const std::string &command)
 
std::string table_type_enum_to_string (const TableType table_type)
 
std::string get_malformed_config_error_message (const std::string &config_key)
 
void validate_expanded_file_path (const std::string &file_path, const std::vector< std::string > &whitelisted_root_paths)
 
std::vector< std::string > get_expanded_file_paths (const std::string &file_path, const DataTransferType data_transfer_type)
 
void validate_allowed_file_path (const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
 
void set_whitelisted_paths (const std::string &config_key, const std::string &config_value, std::vector< std::string > &whitelisted_paths)
 

Enumeration Type Documentation

Enumerator
IMPORT 
EXPORT 

Definition at line 80 of file DdlUtils.h.

enum ddl_utils::TableType
strong
Enumerator
TABLE 
VIEW 
FOREIGN_TABLE 

Definition at line 107 of file DdlUtils.h.

Function Documentation

std::vector<std::string> ddl_utils::get_expanded_file_paths ( const std::string &  file_path,
const DataTransferType  data_transfer_type 
)

Definition at line 746 of file DdlUtils.cpp.

References IMPORT, and shared::local_glob_filter_sort_files().

Referenced by validate_allowed_file_path().

748  {
749  std::vector<std::string> file_paths;
750  if (data_transfer_type == DataTransferType::IMPORT) {
752  file_path, std::nullopt, std::nullopt, std::nullopt);
753  } else {
754  std::string path;
755  if (!boost::filesystem::exists(file_path)) {
756  // For exports, it is possible to provide a path to a new (nonexistent) file. In
757  // this case, validate using the parent path.
758  path = boost::filesystem::path(file_path).parent_path().string();
759  if (!boost::filesystem::exists(path)) {
760  throw std::runtime_error{"File or directory \"" + file_path +
761  "\" does not exist."};
762  }
763  } else {
764  path = file_path;
765  }
766  file_paths = {path};
767  }
768  return file_paths;
769 }
std::vector< std::string > local_glob_filter_sort_files(const std::string &file_path, const std::optional< std::string > &filter_regex, const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex, const bool recurse)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string ddl_utils::get_malformed_config_error_message ( const std::string &  config_key)

Definition at line 723 of file DdlUtils.cpp.

Referenced by set_whitelisted_paths().

723  {
724  return "Configuration value for \"" + config_key +
725  "\" is malformed. Value should be a list of paths with format: [ "
726  "\"root-path-1\", \"root-path-2\", ... ]";
727 }

+ Here is the caller graph for this function:

void ddl_utils::set_column_descriptor ( const std::string &  column_name,
ColumnDescriptor cd,
SqlType *  column_type,
const bool  not_null,
const Encoding *  encoding,
const std::string *  default_value 
)

Definition at line 646 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, SQLTypeInfo::set_notnull(), validate_and_set_array_size(), validate_and_set_default_value(), validate_and_set_encoding(), and validate_and_set_type().

Referenced by Parser::DDLStmt::setColumnDescriptor(), and CreateForeignTableCommand::setColumnDetails().

651  {
652  cd.columnName = column_name;
653  validate_and_set_type(cd, column_type);
654  cd.columnType.set_notnull(not_null);
655  validate_and_set_encoding(cd, encoding, column_type);
656  validate_and_set_array_size(cd, column_type);
657  cd.isSystemCol = false;
658  cd.isVirtualCol = false;
659  validate_and_set_default_value(cd, default_value, not_null);
660 }
void validate_and_set_array_size(ColumnDescriptor &cd, const SqlType *column_type)
Definition: DdlUtils.cpp:487
void validate_and_set_encoding(ColumnDescriptor &cd, const Encoding *encoding, const SqlType *column_type)
Definition: DdlUtils.cpp:432
void set_notnull(bool n)
Definition: sqltypes.h:436
void validate_and_set_default_value(ColumnDescriptor &cd, const std::string *default_value, bool not_null)
Definition: DdlUtils.cpp:625
SQLTypeInfo columnType
std::string columnName
void validate_and_set_type(ColumnDescriptor &cd, SqlType *column_type)
Definition: DdlUtils.cpp:468

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::set_default_encoding ( ColumnDescriptor cd)

Definition at line 213 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, g_use_date_in_days_default_encoding, SQLTypeInfo::get_output_srid(), SQLTypeInfo::get_precision(), SQLTypeInfo::get_type(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_geometry(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kDATE, kENCODING_DATE_IN_DAYS, kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, SQLTypeInfo::set_comp_param(), and SQLTypeInfo::set_compression().

Referenced by validate_and_set_encoding().

213  {
214  // Change default TEXT column behaviour to be DICT encoded
215  if (cd.columnType.is_string() || cd.columnType.is_string_array()) {
216  // default to 32-bits
218  cd.columnType.set_comp_param(32);
219  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 4) {
221  cd.columnType.set_comp_param(16);
222  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() <= 9) {
224  cd.columnType.set_comp_param(32);
225  } else if (cd.columnType.is_decimal() && cd.columnType.get_precision() > 18) {
226  throw std::runtime_error(cd.columnName + ": Precision too high, max 18.");
227  } else if (cd.columnType.is_geometry() && cd.columnType.get_output_srid() == 4326) {
228  // default to GEOINT 32-bits
230  cd.columnType.set_comp_param(32);
232  // Days encoding for DATE
235  } else {
238  }
239 }
void set_compression(EncodingType c)
Definition: sqltypes.h:440
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
bool g_use_date_in_days_default_encoding
Definition: DdlUtils.cpp:35
int get_precision() const
Definition: sqltypes.h:332
void set_comp_param(int p)
Definition: sqltypes.h:441
Definition: sqltypes.h:53
bool is_geometry() const
Definition: sqltypes.h:522
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:510
bool is_string_array() const
Definition: sqltypes.h:511
bool is_decimal() const
Definition: sqltypes.h:513
std::string columnName
HOST DEVICE int get_output_srid() const
Definition: sqltypes.h:335

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::set_default_table_attributes ( const std::string &  table_name,
TableDescriptor td,
const int32_t  column_count 
)

Definition at line 662 of file DdlUtils.cpp.

References DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragmenter, TableDescriptor::fragPageSize, TableDescriptor::fragType, Fragmenter_Namespace::INSERT_ORDER, TableDescriptor::isView, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, and TableDescriptor::tableName.

Referenced by Parser::CreateTableStmt::executeDryRun(), and CreateForeignTableCommand::setTableDetails().

664  {
665  td.tableName = table_name;
666  td.nColumns = column_count;
667  td.isView = false;
668  td.fragmenter = nullptr;
674 }
std::string tableName
#define DEFAULT_MAX_CHUNK_SIZE
#define DEFAULT_MAX_ROWS
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define DEFAULT_PAGE_SIZE
#define DEFAULT_FRAGMENT_ROWS
Fragmenter_Namespace::FragmenterType fragType

+ Here is the caller graph for this function:

void ddl_utils::set_whitelisted_paths ( const std::string &  config_key,
const std::string &  config_value,
std::vector< std::string > &  whitelisted_paths 
)

Definition at line 804 of file DdlUtils.cpp.

References get_malformed_config_error_message(), logger::INFO, LOG, and shared::printContainer().

Referenced by ddl_utils::FilePathWhitelist::initialize().

806  {
807  rapidjson::Document whitelisted_root_paths;
808  whitelisted_root_paths.Parse(config_value);
809  if (!whitelisted_root_paths.IsArray()) {
810  throw std::runtime_error{get_malformed_config_error_message(config_key)};
811  }
812  for (const auto& root_path : whitelisted_root_paths.GetArray()) {
813  if (!root_path.IsString()) {
814  throw std::runtime_error{get_malformed_config_error_message(config_key)};
815  }
816  if (!boost::filesystem::exists(root_path.GetString())) {
817  throw std::runtime_error{"Whitelisted root path \"" +
818  std::string{root_path.GetString()} + "\" does not exist."};
819  }
820  whitelisted_paths.emplace_back(
821  boost::filesystem::canonical(root_path.GetString()).string());
822  }
823  LOG(INFO) << "Parsed " << config_key << ": "
824  << shared::printContainer(whitelisted_paths);
825 }
#define LOG(tag)
Definition: Logger.h:217
std::string get_malformed_config_error_message(const std::string &config_key)
Definition: DdlUtils.cpp:723
PrintContainer< CONTAINER > printContainer(CONTAINER &container)
Definition: misc.h:108

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string ddl_utils::table_type_enum_to_string ( const TableType  table_type)

Definition at line 710 of file DdlUtils.cpp.

References FOREIGN_TABLE, TABLE, and VIEW.

710  {
711  if (table_type == ddl_utils::TableType::TABLE) {
712  return "Table";
713  }
714  if (table_type == ddl_utils::TableType::FOREIGN_TABLE) {
715  return "ForeignTable";
716  }
717  if (table_type == ddl_utils::TableType::VIEW) {
718  return "View";
719  }
720  throw std::runtime_error{"Unexpected table type"};
721 }
void ddl_utils::validate_allowed_file_path ( const std::string &  file_path,
const DataTransferType  data_transfer_type,
const bool  allow_wildcards = false 
)

Validates that the given file path is allowed. Validation entails ensuring that given path is not under a blacklisted root path and path is under a whitelisted path, if whitelisted paths have been configured. Also excludes the use of spaces and punctuation other than: . / _ + - = :

Parameters
file_path- file path to validate
data_transfer_type- enum indicating whether validation is for an import or export use case
allow_wildcards- bool indicating if wildcards are allowed

Definition at line 771 of file DdlUtils.cpp.

References get_expanded_file_paths(), ddl_utils::FilePathBlacklist::isBlacklistedPath(), and ddl_utils::FilePathWhitelist::validateWhitelistedFilePath().

Referenced by TableArchiver::dumpTable(), Parser::CopyTableStmt::execute(), Parser::ExportQueryStmt::execute(), TableArchiver::restoreTable(), anonymous_namespace{TableArchiver.cpp}::simple_file_cat(), anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local(), and foreign_storage::AbstractFileStorageDataWrapper::validateFilePath().

773  {
774  // Reject any punctuation characters except for a few safe ones.
775  // Some punctuation characters present a security risk when passed
776  // to subprocesses. Don't change this without a security review.
777  static const std::string safe_punctuation{"./_+-=:~"};
778  for (const auto& ch : file_path) {
779  if (std::ispunct(ch) && safe_punctuation.find(ch) == std::string::npos &&
780  !(allow_wildcards && ch == '*')) {
781  throw std::runtime_error(std::string("Punctuation \"") + ch +
782  "\" is not allowed in file path: " + file_path);
783  }
784  }
785 
786  // Enforce our whitelist and blacklist for file paths.
787  const auto& expanded_file_paths =
788  get_expanded_file_paths(file_path, data_transfer_type);
789  for (const auto& path : expanded_file_paths) {
790  if (FilePathBlacklist::isBlacklistedPath(path)) {
791  const auto& canonical_file_path = boost::filesystem::canonical(file_path);
792  if (canonical_file_path == boost::filesystem::absolute(file_path)) {
793  throw std::runtime_error{"Access to file or directory path \"" + file_path +
794  "\" is not allowed."};
795  }
796  throw std::runtime_error{"Access to file or directory path \"" + file_path +
797  "\" (resolved to \"" + canonical_file_path.string() +
798  "\") is not allowed."};
799  }
800  }
801  FilePathWhitelist::validateWhitelistedFilePath(expanded_file_paths, data_transfer_type);
802 }
std::vector< std::string > get_expanded_file_paths(const std::string &file_path, const DataTransferType data_transfer_type)
Definition: DdlUtils.cpp:746

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_array_size ( ColumnDescriptor cd,
const SqlType *  column_type 
)

Definition at line 487 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, ddl_utils::SqlType::get_array_size(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_elem_type(), ddl_utils::SqlType::get_is_array(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SQLTypeInfo::set_fixed_size(), and SQLTypeInfo::set_size().

Referenced by set_column_descriptor().

487  {
488  if (cd.columnType.is_string_array() &&
490  throw std::runtime_error(
491  cd.columnName +
492  ": Array of strings must be dictionary encoded. Specify ENCODING DICT");
493  }
494 
495  if (column_type->get_is_array()) {
496  int s = -1;
497  auto array_size = column_type->get_array_size();
498  if (array_size > 0) {
499  auto sti = cd.columnType.get_elem_type();
500  s = array_size * sti.get_size();
501  if (s <= 0) {
502  throw std::runtime_error(cd.columnName + ": Unexpected fixed length array size");
503  }
504  }
505  cd.columnType.set_size(s);
506 
507  } else {
509  }
510 }
void set_size(int s)
Definition: sqltypes.h:437
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
void set_fixed_size()
Definition: sqltypes.h:438
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
SQLTypeInfo columnType
bool is_string_array() const
Definition: sqltypes.h:511
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:865
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_compressed_encoding ( ColumnDescriptor cd,
int  encoding_size 
)

Definition at line 393 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_output_srid(), SQLTypeInfo::is_geometry(), kENCODING_GEOINT, SQLTypeInfo::set_comp_param(), and SQLTypeInfo::set_compression().

Referenced by validate_and_set_encoding().

393  {
394  if (!cd.columnType.is_geometry() || cd.columnType.get_output_srid() != 4326) {
395  throw std::runtime_error(
396  cd.columnName + ": COMPRESSED encoding is only supported on WGS84 geo columns.");
397  }
398  int comp_param;
399  if (encoding_size == 0) {
400  comp_param = 32; // default to 32-bits
401  } else {
402  comp_param = encoding_size;
403  }
404  if (comp_param != 32) {
405  throw std::runtime_error(cd.columnName +
406  ": only 32-bit COMPRESSED geo encoding is supported");
407  }
408  // encoding longitude/latitude as integers
410  cd.columnType.set_comp_param(comp_param);
411 }
void set_compression(EncodingType c)
Definition: sqltypes.h:440
void set_comp_param(int p)
Definition: sqltypes.h:441
bool is_geometry() const
Definition: sqltypes.h:522
SQLTypeInfo columnType
std::string columnName
HOST DEVICE int get_output_srid() const
Definition: sqltypes.h:335

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_date_encoding ( ColumnDescriptor cd,
int  encoding_size 
)

Definition at line 413 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), kARRAY, kDATE, kENCODING_DATE_IN_DAYS, SQLTypeInfo::set_comp_param(), and SQLTypeInfo::set_compression().

Referenced by validate_and_set_encoding().

413  {
414  // days encoding for dates
415  if (cd.columnType.get_type() == kARRAY && cd.columnType.get_subtype() == kDATE) {
416  throw std::runtime_error(cd.columnName +
417  ": Cannot apply days encoding to date array.");
418  }
419  if (cd.columnType.get_type() != kDATE) {
420  throw std::runtime_error(cd.columnName +
421  ": Days encoding is only supported for DATE columns.");
422  }
423  if (encoding_size != 32 && encoding_size != 16) {
424  throw std::runtime_error(cd.columnName +
425  ": Compression parameter for Days encoding on "
426  "DATE must be 16 or 32.");
427  }
429  cd.columnType.set_comp_param((encoding_size == 16) ? 16 : 0);
430 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
void set_compression(EncodingType c)
Definition: sqltypes.h:440
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
void set_comp_param(int p)
Definition: sqltypes.h:441
Definition: sqltypes.h:53
SQLTypeInfo columnType
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_default_value ( ColumnDescriptor cd,
const std::string *  default_value,
bool  not_null 
)

Definition at line 625 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::is_geometry(), to_upper(), and ddl_utils::anonymous_namespace{DdlUtils.cpp}::validate_literal().

Referenced by set_column_descriptor().

627  {
628  bool is_null_literal =
629  default_value && ((to_upper(*default_value) == "NULL") ||
630  (cd.columnType.is_geometry() && default_value->empty()));
631  if (not_null && (is_null_literal)) {
632  throw std::runtime_error(cd.columnName +
633  ": cannot set default value to NULL for "
634  "NOT NULL column");
635  }
636  if (!default_value || is_null_literal) {
637  cd.default_value = std::nullopt;
638  return;
639  }
640  const auto& column_type = cd.columnType;
641  const auto& val = *default_value;
642  validate_literal(val, column_type, cd.columnName);
643  cd.default_value = std::make_optional(*default_value);
644 }
void validate_literal(const std::string &val, SQLTypeInfo column_type, const std::string &column_name)
Definition: DdlUtils.cpp:514
std::string to_upper(const std::string &str)
std::optional< std::string > default_value
bool is_geometry() const
Definition: sqltypes.h:522
SQLTypeInfo columnType
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_dictionary_encoding ( ColumnDescriptor cd,
int  encoding_size 
)

Definition at line 339 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SQLTypeInfo::set_comp_param(), and SQLTypeInfo::set_compression().

Referenced by validate_and_set_encoding().

339  {
340  if (!cd.columnType.is_string() && !cd.columnType.is_string_array()) {
341  throw std::runtime_error(
342  cd.columnName +
343  ": Dictionary encoding is only supported on string or string array columns.");
344  }
345  int comp_param;
346  if (encoding_size == 0) {
347  comp_param = 32; // default to 32-bits
348  } else {
349  comp_param = encoding_size;
350  }
351  if (cd.columnType.is_string_array() && comp_param != 32) {
352  throw std::runtime_error(cd.columnName +
353  ": Compression parameter for string arrays must be 32");
354  }
355  if (comp_param != 8 && comp_param != 16 && comp_param != 32) {
356  throw std::runtime_error(
357  cd.columnName +
358  ": Compression parameter for Dictionary encoding must be 8 or 16 or 32.");
359  }
360  // dictionary encoding
362  cd.columnType.set_comp_param(comp_param);
363 }
void set_compression(EncodingType c)
Definition: sqltypes.h:440
void set_comp_param(int p)
Definition: sqltypes.h:441
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:510
bool is_string_array() const
Definition: sqltypes.h:511
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_encoding ( ColumnDescriptor cd,
const Encoding *  encoding,
const SqlType *  column_type 
)

Definition at line 432 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, ddl_utils::Encoding::get_encoding_name(), ddl_utils::Encoding::get_encoding_param(), kENCODING_DIFF, kENCODING_RL, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), set_default_encoding(), validate_and_set_compressed_encoding(), validate_and_set_date_encoding(), validate_and_set_dictionary_encoding(), validate_and_set_fixed_encoding(), validate_and_set_none_encoding(), and validate_and_set_sparse_encoding().

Referenced by set_column_descriptor().

434  {
435  if (encoding == nullptr) {
437  } else {
438  const std::string& comp = *encoding->get_encoding_name();
439  if (boost::iequals(comp, "fixed")) {
440  validate_and_set_fixed_encoding(cd, encoding->get_encoding_param(), column_type);
441  } else if (boost::iequals(comp, "rl")) {
442  // run length encoding
445  // throw std::runtime_error("RL(Run Length) encoding not supported yet.");
446  } else if (boost::iequals(comp, "diff")) {
447  // differential encoding
450  // throw std::runtime_error("DIFF(differential) encoding not supported yet.");
451  } else if (boost::iequals(comp, "dict")) {
452  validate_and_set_dictionary_encoding(cd, encoding->get_encoding_param());
453  } else if (boost::iequals(comp, "NONE")) {
455  } else if (boost::iequals(comp, "sparse")) {
456  validate_and_set_sparse_encoding(cd, encoding->get_encoding_param());
457  } else if (boost::iequals(comp, "compressed")) {
458  validate_and_set_compressed_encoding(cd, encoding->get_encoding_param());
459  } else if (boost::iequals(comp, "days")) {
460  validate_and_set_date_encoding(cd, encoding->get_encoding_param());
461  } else {
462  throw std::runtime_error(cd.columnName + ": Invalid column compression scheme " +
463  comp);
464  }
465  }
466 }
void set_compression(EncodingType c)
Definition: sqltypes.h:440
void validate_and_set_sparse_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:376
void validate_and_set_dictionary_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:339
void validate_and_set_none_encoding(ColumnDescriptor &cd)
Definition: DdlUtils.cpp:365
void set_default_encoding(ColumnDescriptor &cd)
Definition: DdlUtils.cpp:213
void set_comp_param(int p)
Definition: sqltypes.h:441
void validate_and_set_compressed_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:393
void validate_and_set_fixed_encoding(ColumnDescriptor &cd, int encoding_size, const SqlType *column_type)
Definition: DdlUtils.cpp:241
void validate_and_set_date_encoding(ColumnDescriptor &cd, int encoding_size)
Definition: DdlUtils.cpp:413
SQLTypeInfo columnType
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_fixed_encoding ( ColumnDescriptor cd,
int  encoding_size,
const SqlType *  column_type 
)

Definition at line 241 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_precision(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), is_datetime(), SQLTypeInfo::is_high_precision_timestamp(), IS_INTEGER, kARRAY, kBIGINT, kDATE, kDECIMAL, kENCODING_DATE_IN_DAYS, kENCODING_FIXED, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), ddl_utils::SqlType::to_string(), and run_benchmark_import::type.

Referenced by validate_and_set_encoding().

243  {
244  auto type = cd.columnType.get_type();
245  // fixed-bits encoding
246  if (type == kARRAY) {
247  type = cd.columnType.get_subtype();
248  switch (type) {
249  case kTINYINT:
250  case kSMALLINT:
251  case kINT:
252  case kBIGINT:
253  case kDATE:
254  throw std::runtime_error(cd.columnName + ": Cannot apply FIXED encoding to " +
255  column_type->to_string() + " type array.");
256  break;
257  default:
258  break;
259  }
260  }
261 
262  if (!IS_INTEGER(type) && !is_datetime(type) &&
263  !(type == kDECIMAL || type == kNUMERIC)) {
264  throw std::runtime_error(
265  cd.columnName +
266  ": Fixed encoding is only supported for integer or time columns.");
267  }
268 
269  switch (type) {
270  case kSMALLINT:
271  if (encoding_size != 8) {
272  throw std::runtime_error(
273  cd.columnName +
274  ": Compression parameter for Fixed encoding on SMALLINT must be 8.");
275  }
276  break;
277  case kINT:
278  if (encoding_size != 8 && encoding_size != 16) {
279  throw std::runtime_error(
280  cd.columnName +
281  ": Compression parameter for Fixed encoding on INTEGER must be 8 or 16.");
282  }
283  break;
284  case kBIGINT:
285  if (encoding_size != 8 && encoding_size != 16 && encoding_size != 32) {
286  throw std::runtime_error(cd.columnName +
287  ": Compression parameter for Fixed encoding on "
288  "BIGINT must be 8 or 16 or 32.");
289  }
290  break;
291  case kTIMESTAMP:
292  case kTIME:
293  if (encoding_size != 32) {
294  throw std::runtime_error(cd.columnName +
295  ": Compression parameter for Fixed encoding on "
296  "TIME or TIMESTAMP must be 32.");
297  } else if (cd.columnType.is_high_precision_timestamp()) {
298  throw std::runtime_error("Fixed encoding is not supported for TIMESTAMP(3|6|9).");
299  }
300  break;
301  case kDECIMAL:
302  case kNUMERIC:
303  if (encoding_size != 32 && encoding_size != 16) {
304  throw std::runtime_error(cd.columnName +
305  ": Compression parameter for Fixed encoding on "
306  "DECIMAL must be 16 or 32.");
307  }
308 
309  if (encoding_size == 32 && cd.columnType.get_precision() > 9) {
310  throw std::runtime_error(cd.columnName +
311  ": Precision too high for Fixed(32) encoding, max 9.");
312  }
313 
314  if (encoding_size == 16 && cd.columnType.get_precision() > 4) {
315  throw std::runtime_error(cd.columnName +
316  ": Precision too high for Fixed(16) encoding, max 4.");
317  }
318  break;
319  case kDATE:
320  if (encoding_size != 32 && encoding_size != 16) {
321  throw std::runtime_error(cd.columnName +
322  ": Compression parameter for Fixed encoding on "
323  "DATE must be 16 or 32.");
324  }
325  break;
326  default:
327  throw std::runtime_error(cd.columnName + ": Cannot apply FIXED encoding to " +
328  column_type->to_string());
329  }
330  if (type == kDATE) {
332  cd.columnType.set_comp_param(16);
333  } else {
335  cd.columnType.set_comp_param(encoding_size);
336  }
337 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
void set_compression(EncodingType c)
Definition: sqltypes.h:440
Definition: sqltypes.h:49
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
int get_precision() const
Definition: sqltypes.h:332
void set_comp_param(int p)
Definition: sqltypes.h:441
Definition: sqltypes.h:53
#define IS_INTEGER(T)
Definition: sqltypes.h:245
bool is_high_precision_timestamp() const
Definition: sqltypes.h:885
Definition: sqltypes.h:45
SQLTypeInfo columnType
std::string columnName
constexpr auto is_datetime(SQLTypes type)
Definition: sqltypes.h:263

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_none_encoding ( ColumnDescriptor cd)

Definition at line 365 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_geometry(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_NONE, SQLTypeInfo::set_comp_param(), and SQLTypeInfo::set_compression().

Referenced by validate_and_set_encoding().

365  {
366  if (!cd.columnType.is_string() && !cd.columnType.is_string_array() &&
367  !cd.columnType.is_geometry()) {
368  throw std::runtime_error(
369  cd.columnName +
370  ": None encoding is only supported on string, string array, or geo columns.");
371  }
374 }
void set_compression(EncodingType c)
Definition: sqltypes.h:440
void set_comp_param(int p)
Definition: sqltypes.h:441
bool is_geometry() const
Definition: sqltypes.h:522
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:510
bool is_string_array() const
Definition: sqltypes.h:511
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_sparse_encoding ( ColumnDescriptor cd,
int  encoding_size 
)

Definition at line 376 of file DdlUtils.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_notnull(), kENCODING_SPARSE, SQLTypeInfo::set_comp_param(), and SQLTypeInfo::set_compression().

Referenced by validate_and_set_encoding().

376  {
377  // sparse column encoding with mostly NULL values
378  if (cd.columnType.get_notnull()) {
379  throw std::runtime_error(cd.columnName +
380  ": Cannot do sparse column encoding on a NOT NULL column.");
381  }
382  if (encoding_size == 0 || encoding_size % 8 != 0 || encoding_size > 48) {
383  throw std::runtime_error(
384  cd.columnName +
385  "Must specify number of bits as 8, 16, 24, 32 or 48 as the parameter to "
386  "sparse-column encoding.");
387  }
389  cd.columnType.set_comp_param(encoding_size);
390  // throw std::runtime_error("SPARSE encoding not supported yet.");
391 }
void set_compression(EncodingType c)
Definition: sqltypes.h:440
void set_comp_param(int p)
Definition: sqltypes.h:441
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_and_set_type ( ColumnDescriptor cd,
SqlType *  column_type 
)

Definition at line 468 of file DdlUtils.cpp.

References ddl_utils::SqlType::check_type(), ColumnDescriptor::columnType, ddl_utils::SqlType::get_is_array(), ddl_utils::SqlType::get_param1(), ddl_utils::SqlType::get_param2(), ddl_utils::SqlType::get_type(), IS_GEO, kARRAY, SQLTypeInfo::set_dimension(), SQLTypeInfo::set_input_srid(), SQLTypeInfo::set_output_srid(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_subtype(), and SQLTypeInfo::set_type().

Referenced by set_column_descriptor().

468  {
469  column_type->check_type();
470 
471  if (column_type->get_is_array()) {
473  cd.columnType.set_subtype(column_type->get_type());
474  } else {
475  cd.columnType.set_type(column_type->get_type());
476  }
477  if (IS_GEO(column_type->get_type())) {
478  cd.columnType.set_subtype(static_cast<SQLTypes>(column_type->get_param1()));
479  cd.columnType.set_input_srid(column_type->get_param2());
480  cd.columnType.set_output_srid(column_type->get_param2());
481  } else {
482  cd.columnType.set_dimension(column_type->get_param1());
483  cd.columnType.set_scale(column_type->get_param2());
484  }
485 }
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:430
void set_input_srid(int d)
Definition: sqltypes.h:433
void set_scale(int s)
Definition: sqltypes.h:434
void set_output_srid(int s)
Definition: sqltypes.h:435
void set_dimension(int d)
Definition: sqltypes.h:431
SQLTypeInfo columnType
#define IS_GEO(T)
Definition: sqltypes.h:251
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:429

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ddl_utils::validate_expanded_file_path ( const std::string &  file_path,
const std::vector< std::string > &  whitelisted_root_paths 
)

Definition at line 729 of file DdlUtils.cpp.

Referenced by ddl_utils::FilePathWhitelist::validateWhitelistedFilePath().

730  {
731  const auto& canonical_file_path = boost::filesystem::canonical(file_path);
732  for (const auto& root_path : whitelisted_root_paths) {
733  if (boost::istarts_with(canonical_file_path.string(), root_path)) {
734  return;
735  }
736  }
737  if (canonical_file_path == boost::filesystem::absolute(file_path)) {
738  throw std::runtime_error{"File or directory path \"" + file_path +
739  "\" is not whitelisted."};
740  }
741  throw std::runtime_error{"File or directory path \"" + file_path +
742  "\" (resolved to \"" + canonical_file_path.string() +
743  "\") is not whitelisted."};
744 }

+ Here is the caller graph for this function:

void ddl_utils::validate_non_duplicate_column ( const std::string &  column_name,
std::unordered_set< std::string > &  upper_column_names 
)

Definition at line 676 of file DdlUtils.cpp.

Referenced by Parser::CreateTableStmt::executeDryRun(), and CreateForeignTableCommand::setColumnDetails().

677  {
678  const auto upper_column_name = boost::to_upper_copy<std::string>(column_name);
679  const auto insert_it = upper_column_names.insert(upper_column_name);
680  if (!insert_it.second) {
681  throw std::runtime_error("Column '" + column_name + "' defined more than once");
682  }
683 }

+ Here is the caller graph for this function:

void ddl_utils::validate_non_reserved_keyword ( const std::string &  column_name)

Definition at line 685 of file DdlUtils.cpp.

References reserved_keywords.

Referenced by CreateForeignTableCommand::setColumnDetails().

685  {
686  const auto upper_column_name = boost::to_upper_copy<std::string>(column_name);
687  if (reserved_keywords.find(upper_column_name) != reserved_keywords.end()) {
688  throw std::runtime_error("Cannot create column with reserved keyword '" +
689  column_name + "'");
690  }
691 }
static std::set< std::string > reserved_keywords

+ Here is the caller graph for this function:

void ddl_utils::validate_table_type ( const TableDescriptor td,
const TableType  expected_table_type,
const std::string &  command 
)

Definition at line 693 of file DdlUtils.cpp.

References StorageType::FOREIGN_TABLE, FOREIGN_TABLE, TableDescriptor::isView, TableDescriptor::storageType, TABLE, TableDescriptor::tableName, and VIEW.

Referenced by Parser::AddColumnStmt::check_executable(), DropForeignTableCommand::execute(), AlterForeignTableCommand::execute(), Parser::DropTableStmt::execute(), Parser::RenameColumnStmt::execute(), Parser::DropColumnStmt::execute(), and Parser::DropViewStmt::execute().

695  {
696  if (td->isView) {
697  if (expected_table_type != TableType::VIEW) {
698  throw std::runtime_error(td->tableName + " is a view. Use " + command + " VIEW.");
699  }
700  } else if (td->storageType == StorageType::FOREIGN_TABLE) {
701  if (expected_table_type != TableType::FOREIGN_TABLE) {
702  throw std::runtime_error(td->tableName + " is a foreign table. Use " + command +
703  " FOREIGN TABLE.");
704  }
705  } else if (expected_table_type != TableType::TABLE) {
706  throw std::runtime_error(td->tableName + " is a table. Use " + command + " TABLE.");
707  }
708 }
std::string tableName
std::string storageType
static constexpr char const * FOREIGN_TABLE

+ Here is the caller graph for this function: