OmniSciDB  8fa3bf436f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Parser::CopyTableStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::CopyTableStmt:
+ Collaboration diagram for Parser::CopyTableStmt:

Public Member Functions

 CopyTableStmt (std::string *t, std::string *f, std::list< NameValueAssign * > *o)
 
void execute (const Catalog_Namespace::SessionInfo &session) override
 
void execute (const Catalog_Namespace::SessionInfo &session, const std::function< std::unique_ptr< import_export::Importer >(Catalog_Namespace::Catalog &, const TableDescriptor *, const std::string &, const import_export::CopyParams &)> &importer_factory)
 
std::string & get_table () const
 
bool get_success () const
 
bool was_geo_copy_from () const
 
void get_geo_copy_from_payload (std::string &geo_copy_from_table, std::string &geo_copy_from_file_name, import_export::CopyParams &geo_copy_from_copy_params, std::string &geo_copy_from_partitions)
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

std::unique_ptr< std::string > return_message
 

Private Attributes

std::unique_ptr< std::string > table
 
std::unique_ptr< std::string > file_pattern
 
bool success
 
std::list< std::unique_ptr
< NameValueAssign > > 
options
 
bool _was_geo_copy_from = false
 
std::string _geo_copy_from_file_name
 
import_export::CopyParams _geo_copy_from_copy_params
 
std::string _geo_copy_from_partitions
 

Detailed Description

Definition at line 1481 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::CopyTableStmt::CopyTableStmt ( std::string *  t,
std::string *  f,
std::list< NameValueAssign * > *  o 
)
inline

Definition at line 1483 of file ParserNode.h.

References options.

1484  : table(t), file_pattern(f), success(true) {
1485  if (o) {
1486  for (const auto e : *o) {
1487  options.emplace_back(e);
1488  }
1489  delete o;
1490  }
1491  }
std::list< std::unique_ptr< NameValueAssign > > options
Definition: ParserNode.h:1525
std::unique_ptr< std::string > table
Definition: ParserNode.h:1522
std::unique_ptr< std::string > file_pattern
Definition: ParserNode.h:1523
char * t
char * f

Member Function Documentation

void Parser::CopyTableStmt::execute ( const Catalog_Namespace::SessionInfo session)
overridevirtual

Implements Parser::DDLStmt.

Definition at line 4410 of file ParserNode.cpp.

Referenced by DBHandler::execute_distributed_copy_statement(), omnisci.cursor.Cursor::executemany(), and QueryRunner::QueryRunner::runImport().

4410  {
4411  auto importer_factory = [](Catalog_Namespace::Catalog& catalog,
4412  const TableDescriptor* td,
4413  const std::string& file_path,
4414  const import_export::CopyParams& copy_params) {
4415  return std::make_unique<import_export::Importer>(catalog, td, file_path, copy_params);
4416  };
4417  return execute(session, importer_factory);
4418 }
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:102
void execute(const Catalog_Namespace::SessionInfo &session) override

+ Here is the caller graph for this function:

void Parser::CopyTableStmt::execute ( const Catalog_Namespace::SessionInfo session,
const std::function< std::unique_ptr< import_export::Importer >(Catalog_Namespace::Catalog &, const TableDescriptor *, const std::string &, const import_export::CopyParams &)> &  importer_factory 
)

Definition at line 4420 of file ParserNode.cpp.

References import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, Parser::anonymous_namespace{ParserNode.cpp}::bool_from_string_literal(), import_export::CopyParams::buffer_size, CHECK, import_export::DELIMITED, import_export::CopyParams::delimiter, logger::ERROR, import_export::CopyParams::escape, measure< TimeT >::execution(), legacylockmgr::ExecutorOuterLock, import_export::CopyParams::file_type, g_enable_non_kernel_time_query_interrupt, import_export::CopyParams::geo_assign_render_groups, import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::CopyParams::geo_explode_collections, import_export::CopyParams::geo_layer_name, Catalog_Namespace::SessionInfo::get_currentUser(), Parser::IntLiteral::get_intval(), Catalog_Namespace::SessionInfo::get_session_id(), Parser::StringLiteral::get_stringval(), Catalog_Namespace::SessionInfo::getCatalog(), Executor::getExecutor(), legacylockmgr::LockMgr< MutexType, KeyType >::getMutex(), lockmgr::TableLockMgrImpl< InsertDataLockMgr >::getWriteLockForTable(), import_export::HAS_HEADER, import_export::CopyParams::has_header, ddl_utils::IMPORT, logger::INFO, AccessPrivileges::INSERT_INTO_TABLE, kENCODING_GEOINT, kENCODING_NONE, kGEOMETRY, import_export::CopyParams::line_delim, import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, DBObject::loadKey(), LOG, import_export::CopyParams::lonlat, import_export::CopyParams::max_reject, import_export::NO_HEADER, import_export::CopyParams::null_str, import_export::CopyParams::plain_text, import_export::POLYGON, import_export::CopyParams::quote, import_export::CopyParams::quoted, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::CopyParams::s3_access_key, import_export::CopyParams::s3_endpoint, import_export::CopyParams::s3_region, import_export::CopyParams::s3_secret_key, import_export::CopyParams::s3_session_token, DBObject::setPrivileges(), import_export::CopyParams::source_srid, run_benchmark_import::start_time, test_readcsv::table, TableDBObjectType, TableDescriptor::tableName, import_export::CopyParams::threads, to_string(), toString(), Executor::UNITARY_EXECUTOR_ID, Catalog_Namespace::UserMetadata::userLoggable(), and ddl_utils::validate_allowed_file_path().

Referenced by omnisci.cursor.Cursor::executemany().

4425  {
4426  boost::regex non_local_file_regex{R"(^\s*(s3|http|https)://.+)",
4427  boost::regex::extended | boost::regex::icase};
4428  if (!boost::regex_match(*file_pattern, non_local_file_regex)) {
4431  }
4432 
4433  size_t total_time = 0;
4434 
4435  // Prevent simultaneous import / truncate (see TruncateTableStmt::execute)
4436  const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
4439 
4440  const TableDescriptor* td{nullptr};
4441  std::unique_ptr<lockmgr::TableSchemaLockContainer<lockmgr::ReadLock>> td_with_lock;
4442  std::unique_ptr<lockmgr::WriteLock> insert_data_lock;
4443 
4444  auto& catalog = session.getCatalog();
4445 
4446  try {
4447  td_with_lock = std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::ReadLock>>(
4449  catalog, *table));
4450  td = (*td_with_lock)();
4451  insert_data_lock = std::make_unique<lockmgr::WriteLock>(
4453  } catch (const std::runtime_error& e) {
4454  // noop
4455  // TODO(adb): We're really only interested in whether the table exists or not.
4456  // Create a more refined exception.
4457  }
4458 
4459  // if the table already exists, it's locked, so check access privileges
4460  if (td) {
4461  std::vector<DBObject> privObjects;
4462  DBObject dbObject(*table, TableDBObjectType);
4463  dbObject.loadKey(catalog);
4464  dbObject.setPrivileges(AccessPrivileges::INSERT_INTO_TABLE);
4465  privObjects.push_back(dbObject);
4466  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
4467  throw std::runtime_error("Violation of access privileges: user " +
4468  session.get_currentUser().userLoggable() +
4469  " has no insert privileges for table " + *table + ".");
4470  }
4471  }
4472 
4473  // since we'll have not only posix file names but also s3/hdfs/... url
4474  // we do not expand wildcard or check file existence here.
4475  // from here on, file_path contains something which may be a url
4476  // or a wildcard of file names;
4477  std::string file_path = *file_pattern;
4478  import_export::CopyParams copy_params;
4479  if (!options.empty()) {
4480  for (auto& p : options) {
4481  if (boost::iequals(*p->get_name(), "max_reject")) {
4482  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4483  if (int_literal == nullptr) {
4484  throw std::runtime_error("max_reject option must be an integer.");
4485  }
4486  copy_params.max_reject = int_literal->get_intval();
4487  } else if (boost::iequals(*p->get_name(), "buffer_size")) {
4488  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4489  if (int_literal == nullptr) {
4490  throw std::runtime_error("buffer_size option must be an integer.");
4491  }
4492  copy_params.buffer_size = int_literal->get_intval();
4493  } else if (boost::iequals(*p->get_name(), "threads")) {
4494  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4495  if (int_literal == nullptr) {
4496  throw std::runtime_error("Threads option must be an integer.");
4497  }
4498  copy_params.threads = int_literal->get_intval();
4499  } else if (boost::iequals(*p->get_name(), "delimiter")) {
4500  const StringLiteral* str_literal =
4501  dynamic_cast<const StringLiteral*>(p->get_value());
4502  if (str_literal == nullptr) {
4503  throw std::runtime_error("Delimiter option must be a string.");
4504  } else if (str_literal->get_stringval()->length() != 1) {
4505  throw std::runtime_error("Delimiter must be a single character string.");
4506  }
4507  copy_params.delimiter = (*str_literal->get_stringval())[0];
4508  } else if (boost::iequals(*p->get_name(), "nulls")) {
4509  const StringLiteral* str_literal =
4510  dynamic_cast<const StringLiteral*>(p->get_value());
4511  if (str_literal == nullptr) {
4512  throw std::runtime_error("Nulls option must be a string.");
4513  }
4514  copy_params.null_str = *str_literal->get_stringval();
4515  } else if (boost::iequals(*p->get_name(), "header")) {
4516  const StringLiteral* str_literal =
4517  dynamic_cast<const StringLiteral*>(p->get_value());
4518  if (str_literal == nullptr) {
4519  throw std::runtime_error("Header option must be a boolean.");
4520  }
4521  copy_params.has_header = bool_from_string_literal(str_literal)
4524 #ifdef ENABLE_IMPORT_PARQUET
4525  } else if (boost::iequals(*p->get_name(), "parquet")) {
4526  const StringLiteral* str_literal =
4527  dynamic_cast<const StringLiteral*>(p->get_value());
4528  if (str_literal == nullptr) {
4529  throw std::runtime_error("Parquet option must be a boolean.");
4530  }
4531  if (bool_from_string_literal(str_literal)) {
4532  // not sure a parquet "table" type is proper, but to make code
4533  // look consistent in some places, let's set "table" type too
4534  copy_params.file_type = import_export::FileType::PARQUET;
4535  }
4536 #endif // ENABLE_IMPORT_PARQUET
4537  } else if (boost::iequals(*p->get_name(), "s3_access_key")) {
4538  const StringLiteral* str_literal =
4539  dynamic_cast<const StringLiteral*>(p->get_value());
4540  if (str_literal == nullptr) {
4541  throw std::runtime_error("Option s3_access_key must be a string.");
4542  }
4543  copy_params.s3_access_key = *str_literal->get_stringval();
4544  } else if (boost::iequals(*p->get_name(), "s3_secret_key")) {
4545  const StringLiteral* str_literal =
4546  dynamic_cast<const StringLiteral*>(p->get_value());
4547  if (str_literal == nullptr) {
4548  throw std::runtime_error("Option s3_secret_key must be a string.");
4549  }
4550  copy_params.s3_secret_key = *str_literal->get_stringval();
4551  } else if (boost::iequals(*p->get_name(), "s3_session_token")) {
4552  const StringLiteral* str_literal =
4553  dynamic_cast<const StringLiteral*>(p->get_value());
4554  if (str_literal == nullptr) {
4555  throw std::runtime_error("Option s3_session_token must be a string.");
4556  }
4557  copy_params.s3_session_token = *str_literal->get_stringval();
4558  } else if (boost::iequals(*p->get_name(), "s3_region")) {
4559  const StringLiteral* str_literal =
4560  dynamic_cast<const StringLiteral*>(p->get_value());
4561  if (str_literal == nullptr) {
4562  throw std::runtime_error("Option s3_region must be a string.");
4563  }
4564  copy_params.s3_region = *str_literal->get_stringval();
4565  } else if (boost::iequals(*p->get_name(), "s3_endpoint")) {
4566  const StringLiteral* str_literal =
4567  dynamic_cast<const StringLiteral*>(p->get_value());
4568  if (str_literal == nullptr) {
4569  throw std::runtime_error("Option s3_endpoint must be a string.");
4570  }
4571  copy_params.s3_endpoint = *str_literal->get_stringval();
4572  } else if (boost::iequals(*p->get_name(), "quote")) {
4573  const StringLiteral* str_literal =
4574  dynamic_cast<const StringLiteral*>(p->get_value());
4575  if (str_literal == nullptr) {
4576  throw std::runtime_error("Quote option must be a string.");
4577  } else if (str_literal->get_stringval()->length() != 1) {
4578  throw std::runtime_error("Quote must be a single character string.");
4579  }
4580  copy_params.quote = (*str_literal->get_stringval())[0];
4581  } else if (boost::iequals(*p->get_name(), "escape")) {
4582  const StringLiteral* str_literal =
4583  dynamic_cast<const StringLiteral*>(p->get_value());
4584  if (str_literal == nullptr) {
4585  throw std::runtime_error("Escape option must be a string.");
4586  } else if (str_literal->get_stringval()->length() != 1) {
4587  throw std::runtime_error("Escape must be a single character string.");
4588  }
4589  copy_params.escape = (*str_literal->get_stringval())[0];
4590  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
4591  const StringLiteral* str_literal =
4592  dynamic_cast<const StringLiteral*>(p->get_value());
4593  if (str_literal == nullptr) {
4594  throw std::runtime_error("Line_delimiter option must be a string.");
4595  } else if (str_literal->get_stringval()->length() != 1) {
4596  throw std::runtime_error("Line_delimiter must be a single character string.");
4597  }
4598  copy_params.line_delim = (*str_literal->get_stringval())[0];
4599  } else if (boost::iequals(*p->get_name(), "quoted")) {
4600  const StringLiteral* str_literal =
4601  dynamic_cast<const StringLiteral*>(p->get_value());
4602  if (str_literal == nullptr) {
4603  throw std::runtime_error("Quoted option must be a boolean.");
4604  }
4605  copy_params.quoted = bool_from_string_literal(str_literal);
4606  } else if (boost::iequals(*p->get_name(), "plain_text")) {
4607  const StringLiteral* str_literal =
4608  dynamic_cast<const StringLiteral*>(p->get_value());
4609  if (str_literal == nullptr) {
4610  throw std::runtime_error("plain_text option must be a boolean.");
4611  }
4612  copy_params.plain_text = bool_from_string_literal(str_literal);
4613  } else if (boost::iequals(*p->get_name(), "array_marker")) {
4614  const StringLiteral* str_literal =
4615  dynamic_cast<const StringLiteral*>(p->get_value());
4616  if (str_literal == nullptr) {
4617  throw std::runtime_error("Array Marker option must be a string.");
4618  } else if (str_literal->get_stringval()->length() != 2) {
4619  throw std::runtime_error(
4620  "Array Marker option must be exactly two characters. Default is {}.");
4621  }
4622  copy_params.array_begin = (*str_literal->get_stringval())[0];
4623  copy_params.array_end = (*str_literal->get_stringval())[1];
4624  } else if (boost::iequals(*p->get_name(), "array_delimiter")) {
4625  const StringLiteral* str_literal =
4626  dynamic_cast<const StringLiteral*>(p->get_value());
4627  if (str_literal == nullptr) {
4628  throw std::runtime_error("Array Delimiter option must be a string.");
4629  } else if (str_literal->get_stringval()->length() != 1) {
4630  throw std::runtime_error("Array Delimiter must be a single character string.");
4631  }
4632  copy_params.array_delim = (*str_literal->get_stringval())[0];
4633  } else if (boost::iequals(*p->get_name(), "lonlat")) {
4634  const StringLiteral* str_literal =
4635  dynamic_cast<const StringLiteral*>(p->get_value());
4636  if (str_literal == nullptr) {
4637  throw std::runtime_error("Lonlat option must be a boolean.");
4638  }
4639  copy_params.lonlat = bool_from_string_literal(str_literal);
4640  } else if (boost::iequals(*p->get_name(), "geo")) {
4641  const StringLiteral* str_literal =
4642  dynamic_cast<const StringLiteral*>(p->get_value());
4643  if (str_literal == nullptr) {
4644  throw std::runtime_error("Geo option must be a boolean.");
4645  }
4646  copy_params.file_type = bool_from_string_literal(str_literal)
4649  } else if (boost::iequals(*p->get_name(), "geo_coords_type")) {
4650  const StringLiteral* str_literal =
4651  dynamic_cast<const StringLiteral*>(p->get_value());
4652  if (str_literal == nullptr) {
4653  throw std::runtime_error("'geo_coords_type' option must be a string");
4654  }
4655  const std::string* s = str_literal->get_stringval();
4656  if (boost::iequals(*s, "geography")) {
4657  throw std::runtime_error(
4658  "GEOGRAPHY coords type not yet supported. Please use GEOMETRY.");
4659  // copy_params.geo_coords_type = kGEOGRAPHY;
4660  } else if (boost::iequals(*s, "geometry")) {
4661  copy_params.geo_coords_type = kGEOMETRY;
4662  } else {
4663  throw std::runtime_error(
4664  "Invalid string for 'geo_coords_type' option (must be 'GEOGRAPHY' or "
4665  "'GEOMETRY'): " +
4666  *s);
4667  }
4668  } else if (boost::iequals(*p->get_name(), "geo_coords_encoding")) {
4669  const StringLiteral* str_literal =
4670  dynamic_cast<const StringLiteral*>(p->get_value());
4671  if (str_literal == nullptr) {
4672  throw std::runtime_error("'geo_coords_encoding' option must be a string");
4673  }
4674  const std::string* s = str_literal->get_stringval();
4675  if (boost::iequals(*s, "none")) {
4676  copy_params.geo_coords_encoding = kENCODING_NONE;
4677  copy_params.geo_coords_comp_param = 0;
4678  } else if (boost::iequals(*s, "compressed(32)")) {
4679  copy_params.geo_coords_encoding = kENCODING_GEOINT;
4680  copy_params.geo_coords_comp_param = 32;
4681  } else {
4682  throw std::runtime_error(
4683  "Invalid string for 'geo_coords_encoding' option (must be 'NONE' or "
4684  "'COMPRESSED(32)'): " +
4685  *s);
4686  }
4687  } else if (boost::iequals(*p->get_name(), "geo_coords_srid")) {
4688  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4689  if (int_literal == nullptr) {
4690  throw std::runtime_error("'geo_coords_srid' option must be an integer");
4691  }
4692  const int srid = int_literal->get_intval();
4693  if (srid == 4326 || srid == 3857 || srid == 900913) {
4694  copy_params.geo_coords_srid = srid;
4695  } else {
4696  throw std::runtime_error(
4697  "Invalid value for 'geo_coords_srid' option (must be 4326, 3857, or "
4698  "900913): " +
4699  std::to_string(srid));
4700  }
4701  } else if (boost::iequals(*p->get_name(), "geo_layer_name")) {
4702  const StringLiteral* str_literal =
4703  dynamic_cast<const StringLiteral*>(p->get_value());
4704  if (str_literal == nullptr) {
4705  throw std::runtime_error("'geo_layer_name' option must be a string");
4706  }
4707  const std::string* layer_name = str_literal->get_stringval();
4708  if (layer_name) {
4709  copy_params.geo_layer_name = *layer_name;
4710  } else {
4711  throw std::runtime_error("Invalid value for 'geo_layer_name' option");
4712  }
4713  } else if (boost::iequals(*p->get_name(), "partitions")) {
4714  if (copy_params.file_type == import_export::FileType::POLYGON) {
4715  const auto partitions =
4716  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4717  CHECK(partitions);
4718  const auto partitions_uc = boost::to_upper_copy<std::string>(*partitions);
4719  if (partitions_uc != "REPLICATED") {
4720  throw std::runtime_error("PARTITIONS must be REPLICATED for geo COPY");
4721  }
4722  _geo_copy_from_partitions = partitions_uc;
4723  } else {
4724  throw std::runtime_error("PARTITIONS option not supported for non-geo COPY: " +
4725  *p->get_name());
4726  }
4727  } else if (boost::iequals(*p->get_name(), "geo_assign_render_groups")) {
4728  const StringLiteral* str_literal =
4729  dynamic_cast<const StringLiteral*>(p->get_value());
4730  if (str_literal == nullptr) {
4731  throw std::runtime_error("geo_assign_render_groups option must be a boolean.");
4732  }
4733  copy_params.geo_assign_render_groups = bool_from_string_literal(str_literal);
4734  } else if (boost::iequals(*p->get_name(), "geo_explode_collections")) {
4735  const StringLiteral* str_literal =
4736  dynamic_cast<const StringLiteral*>(p->get_value());
4737  if (str_literal == nullptr) {
4738  throw std::runtime_error("geo_explode_collections option must be a boolean.");
4739  }
4740  copy_params.geo_explode_collections = bool_from_string_literal(str_literal);
4741  } else if (boost::iequals(*p->get_name(), "source_srid")) {
4742  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4743  if (int_literal == nullptr) {
4744  throw std::runtime_error("'source_srid' option must be an integer");
4745  }
4746  const int srid = int_literal->get_intval();
4747  if (copy_params.file_type == import_export::FileType::DELIMITED) {
4748  copy_params.source_srid = srid;
4749  } else {
4750  throw std::runtime_error(
4751  "'source_srid' option can only be used on csv/tsv files");
4752  }
4753  } else {
4754  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
4755  }
4756  }
4757  }
4758 
4759  std::string tr;
4760  if (copy_params.file_type == import_export::FileType::POLYGON) {
4761  // geo import
4762  // we do nothing here, except stash the parameters so we can
4763  // do the import when we unwind to the top of the handler
4764  _geo_copy_from_file_name = file_path;
4765  _geo_copy_from_copy_params = copy_params;
4766  _was_geo_copy_from = true;
4767 
4768  // the result string
4769  // @TODO simon.eves put something more useful in here
4770  // except we really can't because we haven't done the import yet!
4771  if (td) {
4772  tr = std::string("Appending geo to table '") + *table + std::string("'...");
4773  } else {
4774  tr = std::string("Creating table '") + *table +
4775  std::string("' and importing geo...");
4776  }
4777  } else {
4778  if (td) {
4779  CHECK(td_with_lock);
4780 
4781  // regular import
4782  auto importer = importer_factory(catalog, td, file_path, copy_params);
4783  auto start_time = ::toString(std::chrono::system_clock::now());
4784  auto prev_table_epoch = importer->getLoader()->getTableEpochs();
4786  auto query_session = session.get_session_id();
4787  auto query_str = "COPYING " + td->tableName;
4789  executor->enrollQuerySession(query_session,
4790  query_str,
4791  start_time,
4793  QuerySessionStatus::QueryStatus::RUNNING);
4794  }
4795 
4796  ScopeGuard clearInterruptStatus =
4797  [executor, &query_str, &query_session, &start_time, &importer] {
4798  // reset the runtime query interrupt status
4800  executor->clearQuerySessionStatus(query_session, start_time, false);
4801  }
4802  };
4803  import_export::ImportStatus import_result;
4804  auto ms =
4805  measure<>::execution([&]() { import_result = importer->import(&session); });
4806  total_time += ms;
4807  // results
4808  if (!import_result.load_failed &&
4809  import_result.rows_rejected > copy_params.max_reject) {
4810  LOG(ERROR) << "COPY exited early due to reject records count during multi file "
4811  "processing ";
4812  // if we have crossed the truncated load threshold
4813  import_result.load_failed = true;
4814  import_result.load_msg =
4815  "COPY exited early due to reject records count during multi file "
4816  "processing ";
4817  success = false;
4818  }
4819  if (!import_result.load_failed) {
4820  tr = std::string(
4821  "Loaded: " + std::to_string(import_result.rows_completed) +
4822  " recs, Rejected: " + std::to_string(import_result.rows_rejected) +
4823  " recs in " + std::to_string((double)total_time / 1000.0) + " secs");
4824  } else {
4825  tr = std::string("Loader Failed due to : " + import_result.load_msg + " in " +
4826  std::to_string((double)total_time / 1000.0) + " secs");
4827  }
4828  } else {
4829  throw std::runtime_error("Table '" + *table + "' must exist before COPY FROM");
4830  }
4831  }
4832 
4833  return_message.reset(new std::string(tr));
4834  LOG(INFO) << tr;
4835 } // namespace Parser
import_export::CopyParams _geo_copy_from_copy_params
Definition: ParserNode.h:1529
std::string s3_secret_key
Definition: CopyParams.h:63
std::string toString(const ExtArgumentType &sig_type)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::list< std::unique_ptr< NameValueAssign > > options
Definition: ParserNode.h:1525
#define LOG(tag)
Definition: Logger.h:194
std::unique_ptr< std::string > return_message
Definition: ParserNode.h:1499
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:155
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:163
std::unique_ptr< std::string > table
Definition: ParserNode.h:1522
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:115
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
Definition: ParserNode.cpp:921
std::string to_string(char const *&&v)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:163
ImportHeaderRow has_header
Definition: CopyParams.h:48
std::string _geo_copy_from_partitions
Definition: ParserNode.h:1530
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:634
std::string get_session_id() const
Definition: SessionInfo.h:77
std::string geo_layer_name
Definition: CopyParams.h:79
Catalog & getCatalog() const
Definition: SessionInfo.h:66
std::string s3_session_token
Definition: CopyParams.h:64
static std::shared_ptr< MutexType > getMutex(const LockType lockType, const KeyType &key)
Definition: LegacyLockMgr.h:51
std::unique_ptr< std::string > file_pattern
Definition: ParserNode.h:1523
#define CHECK(condition)
Definition: Logger.h:203
std::string _geo_copy_from_file_name
Definition: ParserNode.h:1528
std::string userLoggable() const
Definition: SysCatalog.cpp:125
std::string s3_access_key
Definition: CopyParams.h:62
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:72
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:366
EncodingType geo_coords_encoding
Definition: CopyParams.h:74

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Parser::CopyTableStmt::get_geo_copy_from_payload ( std::string &  geo_copy_from_table,
std::string &  geo_copy_from_file_name,
import_export::CopyParams geo_copy_from_copy_params,
std::string &  geo_copy_from_partitions 
)
inline

Definition at line 1510 of file ParserNode.h.

References _geo_copy_from_copy_params, _geo_copy_from_file_name, _geo_copy_from_partitions, _was_geo_copy_from, and table.

1513  {
1514  geo_copy_from_table = *table;
1515  geo_copy_from_file_name = _geo_copy_from_file_name;
1516  geo_copy_from_copy_params = _geo_copy_from_copy_params;
1517  geo_copy_from_partitions = _geo_copy_from_partitions;
1518  _was_geo_copy_from = false;
1519  }
import_export::CopyParams _geo_copy_from_copy_params
Definition: ParserNode.h:1529
std::unique_ptr< std::string > table
Definition: ParserNode.h:1522
std::string _geo_copy_from_partitions
Definition: ParserNode.h:1530
std::string _geo_copy_from_file_name
Definition: ParserNode.h:1528
bool Parser::CopyTableStmt::get_success ( ) const
inline

Definition at line 1506 of file ParserNode.h.

References success.

1506 { return success; }
std::string& Parser::CopyTableStmt::get_table ( ) const
inline

Definition at line 1501 of file ParserNode.h.

References CHECK, and table.

1501  {
1502  CHECK(table);
1503  return *table;
1504  }
std::unique_ptr< std::string > table
Definition: ParserNode.h:1522
#define CHECK(condition)
Definition: Logger.h:203
bool Parser::CopyTableStmt::was_geo_copy_from ( ) const
inline

Definition at line 1508 of file ParserNode.h.

References _was_geo_copy_from.

1508 { return _was_geo_copy_from; }

Member Data Documentation

import_export::CopyParams Parser::CopyTableStmt::_geo_copy_from_copy_params
private

Definition at line 1529 of file ParserNode.h.

Referenced by get_geo_copy_from_payload().

std::string Parser::CopyTableStmt::_geo_copy_from_file_name
private

Definition at line 1528 of file ParserNode.h.

Referenced by get_geo_copy_from_payload().

std::string Parser::CopyTableStmt::_geo_copy_from_partitions
private

Definition at line 1530 of file ParserNode.h.

Referenced by get_geo_copy_from_payload().

bool Parser::CopyTableStmt::_was_geo_copy_from = false
private

Definition at line 1527 of file ParserNode.h.

Referenced by get_geo_copy_from_payload(), and was_geo_copy_from().

std::unique_ptr<std::string> Parser::CopyTableStmt::file_pattern
private

Definition at line 1523 of file ParserNode.h.

std::list<std::unique_ptr<NameValueAssign> > Parser::CopyTableStmt::options
private

Definition at line 1525 of file ParserNode.h.

Referenced by CopyTableStmt().

std::unique_ptr<std::string> Parser::CopyTableStmt::return_message

Definition at line 1499 of file ParserNode.h.

bool Parser::CopyTableStmt::success
private

Definition at line 1524 of file ParserNode.h.

Referenced by get_success(), omnisci.thrift.ttypes.TQueryResult::read(), omnisci.thrift.OmniSci.connect_result::read(), omnisci.thrift.OmniSci.krb5_connect_result::read(), omnisci.thrift.OmniSci.clone_session_result::read(), omnisci.thrift.OmniSci.get_server_status_result::read(), omnisci.thrift.OmniSci.get_status_result::read(), omnisci.thrift.OmniSci.get_hardware_info_result::read(), omnisci.thrift.OmniSci.get_tables_result::read(), omnisci.thrift.OmniSci.get_physical_tables_result::read(), omnisci.thrift.OmniSci.get_views_result::read(), omnisci.thrift.OmniSci.get_tables_meta_result::read(), omnisci.thrift.OmniSci.get_table_details_result::read(), omnisci.thrift.OmniSci.get_internal_table_details_result::read(), omnisci.thrift.OmniSci.get_users_result::read(), omnisci.thrift.OmniSci.get_databases_result::read(), omnisci.thrift.OmniSci.get_version_result::read(), omnisci.thrift.OmniSci.get_heap_profile_result::read(), omnisci.thrift.OmniSci.get_memory_result::read(), omnisci.thrift.OmniSci.get_table_epoch_result::read(), omnisci.thrift.OmniSci.get_table_epoch_by_name_result::read(), omnisci.thrift.OmniSci.get_table_epochs_result::read(), omnisci.thrift.OmniSci.get_session_info_result::read(), omnisci.thrift.OmniSci.sql_execute_result::read(), omnisci.thrift.OmniSci.sql_execute_df_result::read(), omnisci.thrift.OmniSci.sql_execute_gdf_result::read(), omnisci.thrift.OmniSci.sql_validate_result::read(), omnisci.thrift.OmniSci.get_completion_hints_result::read(), omnisci.thrift.OmniSci.render_vega_result::read(), omnisci.thrift.OmniSci.get_result_row_for_pixel_result::read(), omnisci.thrift.OmniSci.get_dashboard_result::read(), omnisci.thrift.OmniSci.get_dashboards_result::read(), omnisci.thrift.OmniSci.create_dashboard_result::read(), omnisci.thrift.OmniSci.get_dashboard_grantees_result::read(), omnisci.thrift.OmniSci.get_link_view_result::read(), omnisci.thrift.OmniSci.create_link_result::read(), omnisci.thrift.OmniSci.detect_column_types_result::read(), omnisci.thrift.OmniSci.import_table_status_result::read(), omnisci.thrift.OmniSci.get_first_geo_file_in_archive_result::read(), omnisci.thrift.OmniSci.get_all_files_in_archive_result::read(), omnisci.thrift.OmniSci.get_layers_in_geo_file_result::read(), omnisci.thrift.OmniSci.query_get_outer_fragment_count_result::read(), omnisci.thrift.OmniSci.check_table_consistency_result::read(), omnisci.thrift.OmniSci.start_query_result::read(), omnisci.thrift.OmniSci.execute_query_step_result::read(), omnisci.thrift.OmniSci.start_render_query_result::read(), omnisci.thrift.OmniSci.execute_next_render_step_result::read(), omnisci.thrift.OmniSci.get_roles_result::read(), omnisci.thrift.OmniSci.get_db_objects_for_grantee_result::read(), omnisci.thrift.OmniSci.get_db_object_privs_result::read(), omnisci.thrift.OmniSci.get_all_roles_for_user_result::read(), omnisci.thrift.OmniSci.has_role_result::read(), omnisci.thrift.OmniSci.has_object_privilege_result::read(), omnisci.thrift.OmniSci.set_license_key_result::read(), omnisci.thrift.OmniSci.get_license_claims_result::read(), omnisci.thrift.OmniSci.get_device_parameters_result::read(), omnisci.thrift.ttypes.TQueryResult::write(), omnisci.thrift.OmniSci.connect_result::write(), omnisci.thrift.OmniSci.krb5_connect_result::write(), omnisci.thrift.OmniSci.clone_session_result::write(), omnisci.thrift.OmniSci.get_server_status_result::write(), omnisci.thrift.OmniSci.get_status_result::write(), omnisci.thrift.OmniSci.get_hardware_info_result::write(), omnisci.thrift.OmniSci.get_tables_result::write(), omnisci.thrift.OmniSci.get_physical_tables_result::write(), omnisci.thrift.OmniSci.get_views_result::write(), omnisci.thrift.OmniSci.get_tables_meta_result::write(), omnisci.thrift.OmniSci.get_table_details_result::write(), omnisci.thrift.OmniSci.get_internal_table_details_result::write(), omnisci.thrift.OmniSci.get_users_result::write(), omnisci.thrift.OmniSci.get_databases_result::write(), omnisci.thrift.OmniSci.get_version_result::write(), omnisci.thrift.OmniSci.get_heap_profile_result::write(), omnisci.thrift.OmniSci.get_memory_result::write(), omnisci.thrift.OmniSci.get_table_epoch_result::write(), omnisci.thrift.OmniSci.get_table_epoch_by_name_result::write(), omnisci.thrift.OmniSci.get_table_epochs_result::write(), omnisci.thrift.OmniSci.get_session_info_result::write(), omnisci.thrift.OmniSci.sql_execute_result::write(), omnisci.thrift.OmniSci.sql_execute_df_result::write(), omnisci.thrift.OmniSci.sql_execute_gdf_result::write(), omnisci.thrift.OmniSci.sql_validate_result::write(), omnisci.thrift.OmniSci.get_completion_hints_result::write(), omnisci.thrift.OmniSci.render_vega_result::write(), omnisci.thrift.OmniSci.get_result_row_for_pixel_result::write(), omnisci.thrift.OmniSci.get_dashboard_result::write(), omnisci.thrift.OmniSci.get_dashboards_result::write(), omnisci.thrift.OmniSci.create_dashboard_result::write(), omnisci.thrift.OmniSci.get_dashboard_grantees_result::write(), omnisci.thrift.OmniSci.get_link_view_result::write(), omnisci.thrift.OmniSci.create_link_result::write(), omnisci.thrift.OmniSci.detect_column_types_result::write(), omnisci.thrift.OmniSci.import_table_status_result::write(), omnisci.thrift.OmniSci.get_first_geo_file_in_archive_result::write(), omnisci.thrift.OmniSci.get_all_files_in_archive_result::write(), omnisci.thrift.OmniSci.get_layers_in_geo_file_result::write(), omnisci.thrift.OmniSci.query_get_outer_fragment_count_result::write(), omnisci.thrift.OmniSci.check_table_consistency_result::write(), omnisci.thrift.OmniSci.start_query_result::write(), omnisci.thrift.OmniSci.execute_query_step_result::write(), omnisci.thrift.OmniSci.start_render_query_result::write(), omnisci.thrift.OmniSci.execute_next_render_step_result::write(), omnisci.thrift.OmniSci.get_roles_result::write(), omnisci.thrift.OmniSci.get_db_objects_for_grantee_result::write(), omnisci.thrift.OmniSci.get_db_object_privs_result::write(), omnisci.thrift.OmniSci.get_all_roles_for_user_result::write(), omnisci.thrift.OmniSci.has_role_result::write(), omnisci.thrift.OmniSci.has_object_privilege_result::write(), omnisci.thrift.OmniSci.set_license_key_result::write(), omnisci.thrift.OmniSci.get_license_claims_result::write(), and omnisci.thrift.OmniSci.get_device_parameters_result::write().

std::unique_ptr<std::string> Parser::CopyTableStmt::table
private

Definition at line 1522 of file ParserNode.h.

Referenced by get_geo_copy_from_payload(), and get_table().


The documentation for this class was generated from the following files: