OmniSciDB  ba1bac9284
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
QueryRunner::ImportDriver Class Reference

#include <QueryRunner.h>

+ Inheritance diagram for QueryRunner::ImportDriver:
+ Collaboration diagram for QueryRunner::ImportDriver:

Public Member Functions

 ImportDriver (std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU, const std::string session_id="")
 
void importGeoTable (const std::string &file_path, const std::string &table_name, const bool compression, const bool create_table, const bool explode_collections)
 
- Public Member Functions inherited from QueryRunner::QueryRunner
std::shared_ptr
< Catalog_Namespace::SessionInfo
getSession () const
 
void addSessionId (const std::string &session_id, ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
 
void clearSessionId ()
 
std::shared_ptr
< Catalog_Namespace::Catalog
getCatalog () const
 
std::shared_ptr< CalcitegetCalcite () const
 
std::shared_ptr< ExecutorgetExecutor () const
 
Catalog_Namespace::UserMetadatagetUserMetadata () const
 
bool gpusPresent () const
 
virtual void clearGpuMemory () const
 
virtual void clearCpuMemory () const
 
virtual void runDDLStatement (const std::string &)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
 
virtual std::shared_ptr
< ExecutionResult
runSelectQuery (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals=true, const bool allow_loop_joins=true)
 
virtual std::shared_ptr
< ExecutionResult
runSelectQuery (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain=false)
 
virtual std::shared_ptr
< ResultSet
runSQLWithAllowingInterrupt (const std::string &query_str, const std::string &session_id, const ExecutorDeviceType device_type, const double running_query_check_freq=0.9, const unsigned pending_query_check_freq=1000)
 
virtual std::vector
< std::shared_ptr< ResultSet > > 
runMultipleStatements (const std::string &, const ExecutorDeviceType)
 
virtual RegisteredQueryHint getParsedQueryHint (const std::string &)
 
virtual void runImport (Parser::CopyTableStmt *import_stmt)
 
virtual std::unique_ptr
< import_export::Loader
getLoader (const TableDescriptor *td) const
 
const int32_t * getCachedJoinHashTable (size_t idx)
 
const int8_t * getCachedBaselineHashTable (size_t idx)
 
size_t getEntryCntCachedBaselineHashTable (size_t idx)
 
size_t getNumberOfCachedJoinHashTables ()
 
size_t getNumberOfCachedBaselineJoinHashTables ()
 
size_t getNumberOfCachedOverlapsHashTables ()
 
void resizeDispatchQueue (const size_t num_executors)
 
 QueryRunner (std::unique_ptr< Catalog_Namespace::SessionInfo > session)
 
virtual ~QueryRunner ()
 

Additional Inherited Members

- Static Public Member Functions inherited from QueryRunner::QueryRunner
static QueryRunnerinit (const char *db_path, const std::string &udf_filename="", const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20)
 
static QueryRunnerinit (const File_Namespace::DiskCacheConfig *disk_cache_config, const char *db_path, const std::vector< LeafHostInfo > &string_servers={}, const std::vector< LeafHostInfo > &leaf_servers={})
 
static QueryRunnerinit (const char *db_path, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers)
 
static QueryRunnerinit (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename="", bool uses_gpus=true, const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20, const bool create_user=false, const bool create_db=false, const File_Namespace::DiskCacheConfig *config=nullptr)
 
static QueryRunnerinit (std::unique_ptr< Catalog_Namespace::SessionInfo > &session)
 
static QueryRunnerget ()
 
static void reset ()
 
static ExecutionOptions defaultExecutionOptionsForRunSQL (bool allow_loop_joins=true, bool just_explain=false)
 
template<typename... Ts>
static std::shared_ptr
< query_state::QueryState
create_query_state (Ts &&...args)
 
- Static Public Attributes inherited from QueryRunner::QueryRunner
static query_state::QueryStates query_states_
 
- Protected Member Functions inherited from QueryRunner::QueryRunner
 QueryRunner (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename, bool uses_gpus, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, const bool create_db, const File_Namespace::DiskCacheConfig *disk_cache_config=nullptr)
 
- Protected Attributes inherited from QueryRunner::QueryRunner
std::shared_ptr
< Catalog_Namespace::SessionInfo
session_info_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 
std::shared_ptr
< Data_Namespace::DataMgr
data_mgr_
 
- Static Protected Attributes inherited from QueryRunner::QueryRunner
static std::unique_ptr
< QueryRunner
qr_instance_ = nullptr
 

Detailed Description

Definition at line 219 of file QueryRunner.h.

Constructor & Destructor Documentation

QueryRunner::ImportDriver::ImportDriver ( std::shared_ptr< Catalog_Namespace::Catalog cat,
const Catalog_Namespace::UserMetadata user,
const ExecutorDeviceType  dt = ExecutorDeviceType::GPU,
const std::string  session_id = "" 
)

Definition at line 753 of file QueryRunner.cpp.

757  : QueryRunner(
758  std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, session_id)) {}
std::string cat(Ts &&...args)
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)

Member Function Documentation

void QueryRunner::ImportDriver::importGeoTable ( const std::string &  file_path,
const std::string &  table_name,
const bool  compression,
const bool  create_table,
const bool  explode_collections 
)

Definition at line 760 of file QueryRunner.cpp.

References cat(), CHECK, measure< TimeT >::execution(), import_export::CopyParams::geo_coords_encoding, import_export::Importer::importGDAL(), logger::INFO, join(), kDECIMAL, kENCODING_GEOINT, kENCODING_NONE, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, LOG, OMNISCI_GEO_PREFIX, QueryRunner::QueryRunner::runDDLStatement(), ImportHelpers::sanitize_name(), and QueryRunner::QueryRunner::session_info_.

764  {
765  using namespace import_export;
766 
768  const std::string geo_column_name(OMNISCI_GEO_PREFIX);
769 
770  CopyParams copy_params;
771  if (compression) {
773  copy_params.geo_coords_comp_param = 32;
774  } else {
775  copy_params.geo_coords_encoding = EncodingType::kENCODING_NONE;
776  copy_params.geo_coords_comp_param = 0;
777  }
778  copy_params.geo_assign_render_groups = true;
779  copy_params.geo_explode_collections = explode_collections;
780 
781  auto cds = Importer::gdalToColumnDescriptors(file_path, geo_column_name, copy_params);
782  std::map<std::string, std::string> colname_to_src;
783  for (auto& cd : cds) {
784  const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
785  const auto ret =
786  colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
787  CHECK(ret.second);
788  cd.columnName = col_name_sanitized;
789  }
790 
791  auto& cat = session_info_->getCatalog();
792 
793  if (create_table) {
794  const auto td = cat.getMetadataForTable(table_name);
795  if (td != nullptr) {
796  throw std::runtime_error("Error: Table " + table_name +
797  " already exists. Possible failure to correctly re-create "
798  "mapd_data directory.");
799  }
800  if (table_name != ImportHelpers::sanitize_name(table_name)) {
801  throw std::runtime_error("Invalid characters in table name: " + table_name);
802  }
803 
804  std::string stmt{"CREATE TABLE " + table_name};
805  std::vector<std::string> col_stmts;
806 
807  for (auto& cd : cds) {
808  if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
809  cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
810  throw std::runtime_error(
811  "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
812  cd.columnName + " (table: " + table_name + ")");
813  }
814 
815  if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
816  if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
817  cd.columnType.set_precision(14);
818  cd.columnType.set_scale(7);
819  }
820  }
821 
822  std::string col_stmt;
823  col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");
824 
825  if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
826  col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
827  } else {
828  if (cd.columnType.is_string()) {
829  col_stmt.append("ENCODING NONE");
830  } else if (cd.columnType.is_geometry()) {
831  if (cd.columnType.get_output_srid() == 4326) {
832  col_stmt.append("ENCODING NONE");
833  }
834  }
835  }
836  col_stmts.push_back(col_stmt);
837  }
838 
839  stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
840  runDDLStatement(stmt);
841 
842  LOG(INFO) << "Created table: " << table_name;
843  } else {
844  LOG(INFO) << "Not creating table: " << table_name;
845  }
846 
847  const auto td = cat.getMetadataForTable(table_name);
848  if (td == nullptr) {
849  throw std::runtime_error("Error: Failed to create table " + table_name);
850  }
851 
852  import_export::Importer importer(cat, td, file_path, copy_params);
853  auto ms = measure<>::execution(
854  [&]() { importer.importGDAL(colname_to_src, session_info_.get()); });
855  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
856 }
std::string cat(Ts &&...args)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:200
std::string join(T const &container, std::string const &delim)
virtual void runDDLStatement(const std::string &)
std::string sanitize_name(const std::string &name)
const std::string OMNISCI_GEO_PREFIX
Definition: Transforms.h:22
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:214
#define CHECK(condition)
Definition: Logger.h:206
EncodingType geo_coords_encoding
Definition: CopyParams.h:74

+ Here is the call graph for this function:


The documentation for this class was generated from the following files: