OmniSciDB  04ee39c94c
Importer_NS::ImportDriver Class Reference

#include <Importer.h>

+ Inheritance diagram for Importer_NS::ImportDriver:
+ Collaboration diagram for Importer_NS::ImportDriver:

Public Member Functions

 ImportDriver (std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU)
 
void importGeoTable (const std::string &file_path, const std::string &table_name, const bool compression=true, const bool create_table=true)
 
- Public Member Functions inherited from QueryRunner::QueryRunner
Catalog_Namespace::SessionInfogetSession () const
 
std::shared_ptr< Catalog_Namespace::CataloggetCatalog () const
 
std::shared_ptr< CalcitegetCalcite () const
 
bool gpusPresent () const
 
virtual void clearGpuMemory () const
 
virtual void clearCpuMemory () const
 
virtual void runDDLStatement (const std::string &)
 
virtual std::shared_ptr< ResultSetrunSQL (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals=true, const bool allow_loop_joins=true)
 
virtual ExecutionResult runSelectQuery (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain=false)
 
virtual std::vector< std::shared_ptr< ResultSet > > runMultipleStatements (const std::string &, const ExecutorDeviceType)
 
virtual void runImport (Parser::CopyTableStmt *import_stmt)
 
virtual std::unique_ptr< Importer_NS::LoadergetLoader (const TableDescriptor *td) const
 
virtual void setIRFilename (const std::string &filename)
 
virtual ~QueryRunner ()
 
 QueryRunner (std::unique_ptr< Catalog_Namespace::SessionInfo > session)
 

Additional Inherited Members

- Static Public Member Functions inherited from QueryRunner::QueryRunner
static QueryRunnerinit (const char *db_path, const std::string &udf_filename="", const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20)
 
static QueryRunnerinit (const char *db_path, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers)
 
static QueryRunnerinit (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename="", bool uses_gpus=true, const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20, const bool create_user=false, const bool create_db=false)
 
static QueryRunnerinit (std::unique_ptr< Catalog_Namespace::SessionInfo > &session)
 
static QueryRunnerget ()
 
static void reset ()
 
- Protected Member Functions inherited from QueryRunner::QueryRunner
 QueryRunner (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename, bool uses_gpus, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, const bool create_db)
 
Planner::RootPlanparsePlanLegacy (const std::string &query_str)
 
Planner::RootPlanparsePlanCalcite (const std::string &query_str)
 
Planner::RootPlanparsePlan (const std::string &query_str)
 
- Protected Attributes inherited from QueryRunner::QueryRunner
std::unique_ptr< Catalog_Namespace::SessionInfosession_info_
 
- Static Protected Attributes inherited from QueryRunner::QueryRunner
static std::unique_ptr< QueryRunnerqr_instance_ = nullptr
 

Detailed Description

Definition at line 976 of file Importer.h.

Constructor & Destructor Documentation

◆ ImportDriver()

Importer_NS::ImportDriver::ImportDriver ( std::shared_ptr< Catalog_Namespace::Catalog cat,
const Catalog_Namespace::UserMetadata user,
const ExecutorDeviceType  dt = ExecutorDeviceType::GPU 
)

Definition at line 4942 of file Importer.cpp.

4945  : QueryRunner(std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, "")) {}
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)

Member Function Documentation

◆ importGeoTable()

void Importer_NS::ImportDriver::importGeoTable ( const std::string &  file_path,
const std::string &  table_name,
const bool  compression = true,
const bool  create_table = true 
)

Definition at line 4947 of file Importer.cpp.

References CHECK, measure< TimeT >::execution(), Importer_NS::Importer::gdalToColumnDescriptors(), Importer_NS::CopyParams::geo_coords_comp_param, Importer_NS::CopyParams::geo_coords_encoding, Importer_NS::Importer::importGDAL(), logger::INFO, join(), kDECIMAL, kENCODING_GEOINT, kENCODING_NONE, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, LOG, OMNISCI_GEO_PREFIX, QueryRunner::QueryRunner::runDDLStatement(), ImportHelpers::sanitize_name(), and QueryRunner::QueryRunner::session_info_.

Referenced by anonymous_namespace{ImportTest.cpp}::import_test_geofile_importer().

4950  {
4952  const std::string geo_column_name(OMNISCI_GEO_PREFIX);
4953 
4954  CopyParams copy_params;
4955  if (compression) {
4956  copy_params.geo_coords_encoding = EncodingType::kENCODING_GEOINT;
4957  copy_params.geo_coords_comp_param = 32;
4958  } else {
4959  copy_params.geo_coords_encoding = EncodingType::kENCODING_NONE;
4960  copy_params.geo_coords_comp_param = 0;
4961  }
4962 
4963  auto cds = Importer::gdalToColumnDescriptors(file_path, geo_column_name, copy_params);
4964  std::map<std::string, std::string> colname_to_src;
4965  for (auto& cd : cds) {
4966  const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
4967  const auto ret =
4968  colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
4969  CHECK(ret.second);
4970  cd.columnName = col_name_sanitized;
4971  }
4972 
4973  auto& cat = session_info_->getCatalog();
4974 
4975  if (create_table) {
4976  const auto td = cat.getMetadataForTable(table_name);
4977  if (td != nullptr) {
4978  throw std::runtime_error("Error: Table " + table_name +
4979  " already exists. Possible failure to correctly re-create "
4980  "mapd_data directory.");
4981  }
4982  if (table_name != ImportHelpers::sanitize_name(table_name)) {
4983  throw std::runtime_error("Invalid characters in table name: " + table_name);
4984  }
4985 
4986  std::string stmt{"CREATE TABLE " + table_name};
4987  std::vector<std::string> col_stmts;
4988 
4989  for (auto& cd : cds) {
4990  if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
4991  cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
4992  throw std::runtime_error(
4993  "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
4994  cd.columnName + " (table: " + table_name + ")");
4995  }
4996 
4997  if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
4998  if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
4999  cd.columnType.set_precision(14);
5000  cd.columnType.set_scale(7);
5001  }
5002  }
5003 
5004  std::string col_stmt;
5005  col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");
5006 
5007  if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
5008  col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
5009  } else {
5010  if (cd.columnType.is_string()) {
5011  col_stmt.append("ENCODING NONE");
5012  } else if (cd.columnType.is_geometry()) {
5013  if (cd.columnType.get_output_srid() == 4326) {
5014  col_stmt.append("ENCODING NONE");
5015  }
5016  }
5017  }
5018  col_stmts.push_back(col_stmt);
5019  }
5020 
5021  stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
5022  runDDLStatement(stmt);
5023 
5024  LOG(INFO) << "Created table: " << table_name;
5025  } else {
5026  LOG(INFO) << "Not creating table: " << table_name;
5027  }
5028 
5029  const auto td = cat.getMetadataForTable(table_name);
5030  if (td == nullptr) {
5031  throw std::runtime_error("Error: Failed to create table " + table_name);
5032  }
5033 
5034  Importer_NS::Importer importer(cat, td, file_path, copy_params);
5035  auto ms = measure<>::execution([&]() { importer.importGDAL(colname_to_src); });
5036  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
5037 }
#define LOG(tag)
Definition: Logger.h:182
std::string join(T const &container, std::string const &delim)
static const std::list< ColumnDescriptor > gdalToColumnDescriptors(const std::string &fileName, const std::string &geoColumnName, const CopyParams &copy_params)
Definition: Importer.cpp:4265
virtual void runDDLStatement(const std::string &)
std::string sanitize_name(const std::string &name)
std::unique_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:177
#define CHECK(condition)
Definition: Logger.h:187
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
const std::string OMNISCI_GEO_PREFIX
Definition: geosupport.h:23
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

The documentation for this class was generated from the following files: