OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer_NS::ImportDriver Class Reference

#include <Importer.h>

+ Inheritance diagram for Importer_NS::ImportDriver:
+ Collaboration diagram for Importer_NS::ImportDriver:

Public Member Functions

 ImportDriver (std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU)
 
void importGeoTable (const std::string &file_path, const std::string &table_name, const bool compression, const bool create_table, const bool explode_collections)
 
- Public Member Functions inherited from QueryRunner::QueryRunner
std::shared_ptr
< Catalog_Namespace::SessionInfo
getSession () const
 
std::shared_ptr
< Catalog_Namespace::Catalog
getCatalog () const
 
std::shared_ptr< CalcitegetCalcite () const
 
bool gpusPresent () const
 
virtual void clearGpuMemory () const
 
virtual void clearCpuMemory () const
 
virtual void runDDLStatement (const std::string &)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals=true, const bool allow_loop_joins=true)
 
virtual ExecutionResult runSelectQuery (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain=false)
 
virtual std::vector
< std::shared_ptr< ResultSet > > 
runMultipleStatements (const std::string &, const ExecutorDeviceType)
 
virtual void runImport (Parser::CopyTableStmt *import_stmt)
 
virtual std::unique_ptr
< Importer_NS::Loader
getLoader (const TableDescriptor *td) const
 
virtual void setIRFilename (const std::string &filename)
 
virtual ~QueryRunner ()
 
 QueryRunner (std::unique_ptr< Catalog_Namespace::SessionInfo > session)
 

Additional Inherited Members

- Static Public Member Functions inherited from QueryRunner::QueryRunner
static QueryRunnerinit (const char *db_path, const std::string &udf_filename="", const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20)
 
static QueryRunnerinit (const char *db_path, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers)
 
static QueryRunnerinit (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename="", bool uses_gpus=true, const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20, const bool create_user=false, const bool create_db=false)
 
static QueryRunnerinit (std::unique_ptr< Catalog_Namespace::SessionInfo > &session)
 
static QueryRunnerget ()
 
static void reset ()
 
template<typename... Ts>
static std::shared_ptr
< query_state::QueryState
create_query_state (Ts &&...args)
 
- Static Public Attributes inherited from QueryRunner::QueryRunner
static query_state::QueryStates query_states_
 
- Protected Member Functions inherited from QueryRunner::QueryRunner
 QueryRunner (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename, bool uses_gpus, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, const bool create_db)
 
Planner::RootPlanparsePlanLegacy (const std::string &query_str)
 
Planner::RootPlanparsePlanCalcite (QueryStateProxy)
 
Planner::RootPlanparsePlan (QueryStateProxy)
 
- Protected Attributes inherited from QueryRunner::QueryRunner
std::shared_ptr
< Catalog_Namespace::SessionInfo
session_info_
 
- Static Protected Attributes inherited from QueryRunner::QueryRunner
static std::unique_ptr
< QueryRunner
qr_instance_ = nullptr
 

Detailed Description

Definition at line 838 of file Importer.h.

Constructor & Destructor Documentation

Importer_NS::ImportDriver::ImportDriver ( std::shared_ptr< Catalog_Namespace::Catalog cat,
const Catalog_Namespace::UserMetadata user,
const ExecutorDeviceType  dt = ExecutorDeviceType::GPU 
)

Definition at line 5034 of file Importer.cpp.

5037  : QueryRunner(std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, "")) {}
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)

Member Function Documentation

void Importer_NS::ImportDriver::importGeoTable ( const std::string &  file_path,
const std::string &  table_name,
const bool  compression,
const bool  create_table,
const bool  explode_collections 
)

Definition at line 5039 of file Importer.cpp.

References CHECK(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, measure< TimeT >::execution(), Importer_NS::Importer::gdalToColumnDescriptors(), Importer_NS::CopyParams::geo_assign_render_groups, Importer_NS::CopyParams::geo_coords_comp_param, Importer_NS::CopyParams::geo_coords_encoding, Importer_NS::CopyParams::geo_explode_collections, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression_name(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_output_srid(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_precision(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type_name(), Importer_NS::Importer::importGDAL(), logger::INFO, SQLTypeInfoCore< TYPE_FACET_PACK >::is_geometry(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string(), join(), kDECIMAL, kENCODING_GEOINT, kENCODING_NONE, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, LOG, OMNISCI_GEO_PREFIX, QueryRunner::QueryRunner::runDDLStatement(), ImportHelpers::sanitize_name(), QueryRunner::QueryRunner::session_info_, SQLTypeInfoCore< TYPE_FACET_PACK >::set_precision(), and SQLTypeInfoCore< TYPE_FACET_PACK >::set_scale().

5043  {
5045  const std::string geo_column_name(OMNISCI_GEO_PREFIX);
5046 
5047  CopyParams copy_params;
5048  if (compression) {
5049  copy_params.geo_coords_encoding = EncodingType::kENCODING_GEOINT;
5050  copy_params.geo_coords_comp_param = 32;
5051  } else {
5052  copy_params.geo_coords_encoding = EncodingType::kENCODING_NONE;
5053  copy_params.geo_coords_comp_param = 0;
5054  }
5055  copy_params.geo_assign_render_groups = true;
5056  copy_params.geo_explode_collections = explode_collections;
5057 
5058  auto cds = Importer::gdalToColumnDescriptors(file_path, geo_column_name, copy_params);
5059  std::map<std::string, std::string> colname_to_src;
5060  for (auto& cd : cds) {
5061  const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
5062  const auto ret =
5063  colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
5064  CHECK(ret.second);
5065  cd.columnName = col_name_sanitized;
5066  }
5067 
5068  auto& cat = session_info_->getCatalog();
5069 
5070  if (create_table) {
5071  const auto td = cat.getMetadataForTable(table_name);
5072  if (td != nullptr) {
5073  throw std::runtime_error("Error: Table " + table_name +
5074  " already exists. Possible failure to correctly re-create "
5075  "mapd_data directory.");
5076  }
5077  if (table_name != ImportHelpers::sanitize_name(table_name)) {
5078  throw std::runtime_error("Invalid characters in table name: " + table_name);
5079  }
5080 
5081  std::string stmt{"CREATE TABLE " + table_name};
5082  std::vector<std::string> col_stmts;
5083 
5084  for (auto& cd : cds) {
5085  if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
5086  cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
5087  throw std::runtime_error(
5088  "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
5089  cd.columnName + " (table: " + table_name + ")");
5090  }
5091 
5092  if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
5093  if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
5094  cd.columnType.set_precision(14);
5095  cd.columnType.set_scale(7);
5096  }
5097  }
5098 
5099  std::string col_stmt;
5100  col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");
5101 
5102  if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
5103  col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
5104  } else {
5105  if (cd.columnType.is_string()) {
5106  col_stmt.append("ENCODING NONE");
5107  } else if (cd.columnType.is_geometry()) {
5108  if (cd.columnType.get_output_srid() == 4326) {
5109  col_stmt.append("ENCODING NONE");
5110  }
5111  }
5112  }
5113  col_stmts.push_back(col_stmt);
5114  }
5115 
5116  stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
5117  runDDLStatement(stmt);
5118 
5119  LOG(INFO) << "Created table: " << table_name;
5120  } else {
5121  LOG(INFO) << "Not creating table: " << table_name;
5122  }
5123 
5124  const auto td = cat.getMetadataForTable(table_name);
5125  if (td == nullptr) {
5126  throw std::runtime_error("Error: Failed to create table " + table_name);
5127  }
5128 
5129  Importer_NS::Importer importer(cat, td, file_path, copy_params);
5130  auto ms = measure<>::execution([&]() { importer.importGDAL(colname_to_src); });
5131  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
5132 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:185
std::string join(T const &container, std::string const &delim)
static const std::list< ColumnDescriptor > gdalToColumnDescriptors(const std::string &fileName, const std::string &geoColumnName, const CopyParams &copy_params)
Definition: Importer.cpp:4334
CHECK(cgen_state)
virtual void runDDLStatement(const std::string &)
std::string sanitize_name(const std::string &name)
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:188
const std::string OMNISCI_GEO_PREFIX
Definition: geosupport.h:23

+ Here is the call graph for this function:


The documentation for this class was generated from the following files: