OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
QueryRunner::ImportDriver Class Reference

#include <QueryRunner.h>

+ Inheritance diagram for QueryRunner::ImportDriver:
+ Collaboration diagram for QueryRunner::ImportDriver:

Public Member Functions

 ImportDriver (std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU)
 
void importGeoTable (const std::string &file_path, const std::string &table_name, const bool compression, const bool create_table, const bool explode_collections)
 
- Public Member Functions inherited from QueryRunner::QueryRunner
std::shared_ptr
< Catalog_Namespace::SessionInfo
getSession () const
 
std::shared_ptr
< Catalog_Namespace::Catalog
getCatalog () const
 
std::shared_ptr< CalcitegetCalcite () const
 
std::shared_ptr< ExecutorgetExecutor () const
 
bool gpusPresent () const
 
virtual void clearGpuMemory () const
 
virtual void clearCpuMemory () const
 
virtual void runDDLStatement (const std::string &)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals=true, const bool allow_loop_joins=true)
 
virtual std::shared_ptr
< ExecutionResult
runSelectQuery (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain=false)
 
virtual std::shared_ptr
< ResultSet
runSQLWithAllowingInterrupt (const std::string &query_str, std::shared_ptr< Executor > executor, const std::string &session_id, const ExecutorDeviceType device_type, const double running_query_check_freq=0.9, const unsigned pending_query_check_freq=1000)
 
virtual std::vector
< std::shared_ptr< ResultSet > > 
runMultipleStatements (const std::string &, const ExecutorDeviceType)
 
virtual QueryHint getParsedQueryHintofQuery (const std::string &)
 
virtual void runImport (Parser::CopyTableStmt *import_stmt)
 
virtual std::unique_ptr
< import_export::Loader
getLoader (const TableDescriptor *td) const
 
const std::shared_ptr
< std::vector< int32_t > > & 
getCachedJoinHashTable (size_t idx)
 
const int8_t * getCachedBaselineHashTable (size_t idx)
 
size_t getEntryCntCachedBaselineHashTable (size_t idx)
 
uint64_t getNumberOfCachedJoinHashTables ()
 
uint64_t getNumberOfCachedBaselineJoinHashTables ()
 
void resizeDispatchQueue (const size_t num_executors)
 
 QueryRunner (std::unique_ptr< Catalog_Namespace::SessionInfo > session)
 
virtual ~QueryRunner ()=default
 

Additional Inherited Members

- Static Public Member Functions inherited from QueryRunner::QueryRunner
static QueryRunnerinit (const char *db_path, const std::string &udf_filename="", const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20)
 
static QueryRunnerinit (const char *db_path, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers)
 
static QueryRunnerinit (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename="", bool uses_gpus=true, const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20, const bool create_user=false, const bool create_db=false)
 
static QueryRunnerinit (std::unique_ptr< Catalog_Namespace::SessionInfo > &session)
 
static QueryRunnerget ()
 
static void reset ()
 
template<typename... Ts>
static std::shared_ptr
< query_state::QueryState
create_query_state (Ts &&...args)
 
- Static Public Attributes inherited from QueryRunner::QueryRunner
static query_state::QueryStates query_states_
 
- Protected Member Functions inherited from QueryRunner::QueryRunner
 QueryRunner (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename, bool uses_gpus, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, const bool create_db)
 
- Protected Attributes inherited from QueryRunner::QueryRunner
std::shared_ptr
< Catalog_Namespace::SessionInfo
session_info_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 
- Static Protected Attributes inherited from QueryRunner::QueryRunner
static std::unique_ptr
< QueryRunner
qr_instance_ = nullptr
 

Detailed Description

Definition at line 192 of file QueryRunner.h.

Constructor & Destructor Documentation

QueryRunner::ImportDriver::ImportDriver ( std::shared_ptr< Catalog_Namespace::Catalog cat,
const Catalog_Namespace::UserMetadata user,
const ExecutorDeviceType  dt = ExecutorDeviceType::GPU 
)

Definition at line 618 of file QueryRunner.cpp.

621  : QueryRunner(std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, "")) {}
std::string cat(Ts &&...args)
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)

Member Function Documentation

void QueryRunner::ImportDriver::importGeoTable ( const std::string &  file_path,
const std::string &  table_name,
const bool  compression,
const bool  create_table,
const bool  explode_collections 
)

Definition at line 623 of file QueryRunner.cpp.

References cat(), CHECK, measure< TimeT >::execution(), import_export::CopyParams::geo_coords_encoding, import_export::Importer::importGDAL(), logger::INFO, join(), kDECIMAL, kENCODING_GEOINT, kENCODING_NONE, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, LOG, OMNISCI_GEO_PREFIX, QueryRunner::QueryRunner::runDDLStatement(), ImportHelpers::sanitize_name(), and QueryRunner::QueryRunner::session_info_.

627  {
628  using namespace import_export;
629 
631  const std::string geo_column_name(OMNISCI_GEO_PREFIX);
632 
633  CopyParams copy_params;
634  if (compression) {
636  copy_params.geo_coords_comp_param = 32;
637  } else {
638  copy_params.geo_coords_encoding = EncodingType::kENCODING_NONE;
639  copy_params.geo_coords_comp_param = 0;
640  }
641  copy_params.geo_assign_render_groups = true;
642  copy_params.geo_explode_collections = explode_collections;
643 
644  auto cds = Importer::gdalToColumnDescriptors(file_path, geo_column_name, copy_params);
645  std::map<std::string, std::string> colname_to_src;
646  for (auto& cd : cds) {
647  const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
648  const auto ret =
649  colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
650  CHECK(ret.second);
651  cd.columnName = col_name_sanitized;
652  }
653 
654  auto& cat = session_info_->getCatalog();
655 
656  if (create_table) {
657  const auto td = cat.getMetadataForTable(table_name);
658  if (td != nullptr) {
659  throw std::runtime_error("Error: Table " + table_name +
660  " already exists. Possible failure to correctly re-create "
661  "mapd_data directory.");
662  }
663  if (table_name != ImportHelpers::sanitize_name(table_name)) {
664  throw std::runtime_error("Invalid characters in table name: " + table_name);
665  }
666 
667  std::string stmt{"CREATE TABLE " + table_name};
668  std::vector<std::string> col_stmts;
669 
670  for (auto& cd : cds) {
671  if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
672  cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
673  throw std::runtime_error(
674  "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
675  cd.columnName + " (table: " + table_name + ")");
676  }
677 
678  if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
679  if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
680  cd.columnType.set_precision(14);
681  cd.columnType.set_scale(7);
682  }
683  }
684 
685  std::string col_stmt;
686  col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");
687 
688  if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
689  col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
690  } else {
691  if (cd.columnType.is_string()) {
692  col_stmt.append("ENCODING NONE");
693  } else if (cd.columnType.is_geometry()) {
694  if (cd.columnType.get_output_srid() == 4326) {
695  col_stmt.append("ENCODING NONE");
696  }
697  }
698  }
699  col_stmts.push_back(col_stmt);
700  }
701 
702  stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
703  runDDLStatement(stmt);
704 
705  LOG(INFO) << "Created table: " << table_name;
706  } else {
707  LOG(INFO) << "Not creating table: " << table_name;
708  }
709 
710  const auto td = cat.getMetadataForTable(table_name);
711  if (td == nullptr) {
712  throw std::runtime_error("Error: Failed to create table " + table_name);
713  }
714 
715  import_export::Importer importer(cat, td, file_path, copy_params);
716  auto ms = measure<>::execution([&]() { importer.importGDAL(colname_to_src); });
717  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
718 }
std::string cat(Ts &&...args)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:188
std::string join(T const &container, std::string const &delim)
virtual void runDDLStatement(const std::string &)
std::string sanitize_name(const std::string &name)
const std::string OMNISCI_GEO_PREFIX
Definition: Transforms.h:22
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:188
#define CHECK(condition)
Definition: Logger.h:197
EncodingType geo_coords_encoding
Definition: CopyParams.h:73

+ Here is the call graph for this function:


The documentation for this class was generated from the following files: