OmniSciDB  eb3a3d0a03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Parser::CopyTableStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::CopyTableStmt:
+ Collaboration diagram for Parser::CopyTableStmt:

Public Member Functions

 CopyTableStmt (std::string *t, std::string *f, std::list< NameValueAssign * > *o)
 
 CopyTableStmt (const rapidjson::Value &payload)
 
void execute (const Catalog_Namespace::SessionInfo &session) override
 
void execute (const Catalog_Namespace::SessionInfo &session, const std::function< std::unique_ptr< import_export::Importer >(Catalog_Namespace::Catalog &, const TableDescriptor *, const std::string &, const import_export::CopyParams &)> &importer_factory)
 
std::string & get_table () const
 
bool get_success () const
 
bool was_geo_copy_from () const
 
void get_geo_copy_from_payload (std::string &geo_copy_from_table, std::string &geo_copy_from_file_name, import_export::CopyParams &geo_copy_from_copy_params, std::string &geo_copy_from_partitions)
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

std::unique_ptr< std::string > return_message
 

Private Attributes

std::unique_ptr< std::string > table_
 
std::unique_ptr< std::string > file_pattern_
 
bool success_
 
std::list< std::unique_ptr
< NameValueAssign > > 
options_
 
bool was_geo_copy_from_ = false
 
std::string geo_copy_from_file_name_
 
import_export::CopyParams geo_copy_from_copy_params_
 
std::string geo_copy_from_partitions_
 

Detailed Description

Definition at line 1472 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::CopyTableStmt::CopyTableStmt ( std::string *  t,
std::string *  f,
std::list< NameValueAssign * > *  o 
)

Definition at line 4595 of file ParserNode.cpp.

References options_.

4598  : table_(t), file_pattern_(f), success_(true) {
4599  if (o) {
4600  for (const auto e : *o) {
4601  options_.emplace_back(e);
4602  }
4603  delete o;
4604  }
4605 }
std::unique_ptr< std::string > file_pattern_
Definition: ParserNode.h:1508
std::unique_ptr< std::string > table_
Definition: ParserNode.h:1507
char * t
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1510
char * f
Parser::CopyTableStmt::CopyTableStmt ( const rapidjson::Value &  payload)

Definition at line 4607 of file ParserNode.cpp.

References CHECK, file_pattern_, json_str(), options_, Parser::anonymous_namespace{ParserNode.cpp}::parse_options(), and table_.

4607  : success_(true) {
4608  CHECK(payload.HasMember("table"));
4609  table_ = std::make_unique<std::string>(json_str(payload["table"]));
4610 
4611  CHECK(payload.HasMember("filePath"));
4612  std::string fs = json_str(payload["filePath"]);
4613  // strip leading/trailing spaces/quotes/single quotes
4614  boost::algorithm::trim_if(fs, boost::is_any_of(" \"'`"));
4615  file_pattern_ = std::make_unique<std::string>(fs);
4616 
4617  parse_options(payload, options_);
4618 }
const std::string json_str(const rapidjson::Value &obj) noexcept
Definition: JsonAccessors.h:44
std::unique_ptr< std::string > file_pattern_
Definition: ParserNode.h:1508
std::unique_ptr< std::string > table_
Definition: ParserNode.h:1507
void parse_options(const rapidjson::Value &payload, std::list< std::unique_ptr< NameValueAssign >> &nameValueList, bool stringToNull=false, bool stringToInteger=false)
#define CHECK(condition)
Definition: Logger.h:209
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1510

+ Here is the call graph for this function:

Member Function Documentation

void Parser::CopyTableStmt::execute ( const Catalog_Namespace::SessionInfo session)
overridevirtual

Implements Parser::DDLStmt.

Definition at line 4620 of file ParserNode.cpp.

Referenced by DBHandler::execute_distributed_copy_statement(), omnisci.cursor.Cursor::executemany(), and QueryRunner::QueryRunner::runImport().

4620  {
4621  auto importer_factory = [](Catalog_Namespace::Catalog& catalog,
4622  const TableDescriptor* td,
4623  const std::string& file_path,
4624  const import_export::CopyParams& copy_params) {
4625  return std::make_unique<import_export::Importer>(catalog, td, file_path, copy_params);
4626  };
4627  return execute(session, importer_factory);
4628 }
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:102
void execute(const Catalog_Namespace::SessionInfo &session) override

+ Here is the caller graph for this function:

void Parser::CopyTableStmt::execute ( const Catalog_Namespace::SessionInfo session,
const std::function< std::unique_ptr< import_export::Importer >(Catalog_Namespace::Catalog &, const TableDescriptor *, const std::string &, const import_export::CopyParams &)> &  importer_factory 
)

Definition at line 4630 of file ParserNode.cpp.

References import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, Parser::anonymous_namespace{ParserNode.cpp}::bool_from_string_literal(), import_export::CopyParams::buffer_size, CHECK, import_export::DELIMITED, import_export::CopyParams::delimiter, logger::ERROR, import_export::CopyParams::escape, measure< TimeT >::execution(), legacylockmgr::ExecutorOuterLock, file_pattern_, import_export::CopyParams::file_type, g_enable_non_kernel_time_query_interrupt, import_export::CopyParams::geo_assign_render_groups, import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, geo_copy_from_copy_params_, geo_copy_from_file_name_, geo_copy_from_partitions_, import_export::CopyParams::geo_explode_collections, import_export::CopyParams::geo_layer_name, Catalog_Namespace::SessionInfo::get_currentUser(), Parser::IntLiteral::get_intval(), Catalog_Namespace::SessionInfo::get_session_id(), Parser::StringLiteral::get_stringval(), Catalog_Namespace::SessionInfo::getCatalog(), Executor::getExecutor(), legacylockmgr::LockMgr< MutexType, KeyType >::getMutex(), lockmgr::TableLockMgrImpl< InsertDataLockMgr >::getWriteLockForTable(), import_export::HAS_HEADER, import_export::CopyParams::has_header, ddl_utils::IMPORT, logger::INFO, AccessPrivileges::INSERT_INTO_TABLE, kENCODING_GEOINT, kENCODING_NONE, kGEOMETRY, import_export::CopyParams::line_delim, import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, DBObject::loadKey(), LOG, import_export::CopyParams::lonlat, import_export::CopyParams::max_reject, import_export::NO_HEADER, import_export::CopyParams::null_str, options_, import_export::CopyParams::plain_text, import_export::POLYGON, import_export::CopyParams::quote, import_export::CopyParams::quoted, import_export::CopyParams::regex_path_filter, return_message, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::CopyParams::s3_access_key, import_export::CopyParams::s3_endpoint, import_export::CopyParams::s3_region, import_export::CopyParams::s3_secret_key, import_export::CopyParams::s3_session_token, DBObject::setPrivileges(), import_export::CopyParams::source_srid, run_benchmark_import::start_time, success_, table_, TableDBObjectType, TableDescriptor::tableName, import_export::CopyParams::threads, to_string(), toString(), Executor::UNITARY_EXECUTOR_ID, Catalog_Namespace::UserMetadata::userLoggable(), ddl_utils::validate_allowed_file_path(), and was_geo_copy_from_.

Referenced by omnisci.cursor.Cursor::executemany().

4635  {
4636  boost::regex non_local_file_regex{R"(^\s*(s3|http|https)://.+)",
4637  boost::regex::extended | boost::regex::icase};
4638  if (!boost::regex_match(*file_pattern_, non_local_file_regex)) {
4641  }
4642 
4643  size_t total_time = 0;
4644 
4645  // Prevent simultaneous import / truncate (see TruncateTableStmt::execute)
4646  const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
4649 
4650  const TableDescriptor* td{nullptr};
4651  std::unique_ptr<lockmgr::TableSchemaLockContainer<lockmgr::ReadLock>> td_with_lock;
4652  std::unique_ptr<lockmgr::WriteLock> insert_data_lock;
4653 
4654  auto& catalog = session.getCatalog();
4655 
4656  try {
4657  td_with_lock = std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::ReadLock>>(
4659  catalog, *table_));
4660  td = (*td_with_lock)();
4661  insert_data_lock = std::make_unique<lockmgr::WriteLock>(
4663  } catch (const std::runtime_error& e) {
4664  // noop
4665  // TODO(adb): We're really only interested in whether the table exists or not.
4666  // Create a more refined exception.
4667  }
4668 
4669  // if the table already exists, it's locked, so check access privileges
4670  if (td) {
4671  std::vector<DBObject> privObjects;
4672  DBObject dbObject(*table_, TableDBObjectType);
4673  dbObject.loadKey(catalog);
4674  dbObject.setPrivileges(AccessPrivileges::INSERT_INTO_TABLE);
4675  privObjects.push_back(dbObject);
4676  if (!SysCatalog::instance().checkPrivileges(session.get_currentUser(), privObjects)) {
4677  throw std::runtime_error("Violation of access privileges: user " +
4678  session.get_currentUser().userLoggable() +
4679  " has no insert privileges for table " + *table_ + ".");
4680  }
4681  }
4682 
4683  // since we'll have not only posix file names but also s3/hdfs/... url
4684  // we do not expand wildcard or check file existence here.
4685  // from here on, file_path contains something which may be a url
4686  // or a wildcard of file names;
4687  std::string file_path = *file_pattern_;
4688  import_export::CopyParams copy_params;
4689  if (!options_.empty()) {
4690  for (auto& p : options_) {
4691  if (boost::iequals(*p->get_name(), "max_reject")) {
4692  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4693  if (int_literal == nullptr) {
4694  throw std::runtime_error("max_reject option must be an integer.");
4695  }
4696  copy_params.max_reject = int_literal->get_intval();
4697  } else if (boost::iequals(*p->get_name(), "buffer_size")) {
4698  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4699  if (int_literal == nullptr) {
4700  throw std::runtime_error("buffer_size option must be an integer.");
4701  }
4702  copy_params.buffer_size = int_literal->get_intval();
4703  } else if (boost::iequals(*p->get_name(), "threads")) {
4704  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4705  if (int_literal == nullptr) {
4706  throw std::runtime_error("Threads option must be an integer.");
4707  }
4708  copy_params.threads = int_literal->get_intval();
4709  } else if (boost::iequals(*p->get_name(), "delimiter")) {
4710  const StringLiteral* str_literal =
4711  dynamic_cast<const StringLiteral*>(p->get_value());
4712  if (str_literal == nullptr) {
4713  throw std::runtime_error("Delimiter option must be a string.");
4714  } else if (str_literal->get_stringval()->length() != 1) {
4715  throw std::runtime_error("Delimiter must be a single character string.");
4716  }
4717  copy_params.delimiter = (*str_literal->get_stringval())[0];
4718  } else if (boost::iequals(*p->get_name(), "nulls")) {
4719  const StringLiteral* str_literal =
4720  dynamic_cast<const StringLiteral*>(p->get_value());
4721  if (str_literal == nullptr) {
4722  throw std::runtime_error("Nulls option must be a string.");
4723  }
4724  copy_params.null_str = *str_literal->get_stringval();
4725  } else if (boost::iequals(*p->get_name(), "header")) {
4726  const StringLiteral* str_literal =
4727  dynamic_cast<const StringLiteral*>(p->get_value());
4728  if (str_literal == nullptr) {
4729  throw std::runtime_error("Header option must be a boolean.");
4730  }
4731  copy_params.has_header = bool_from_string_literal(str_literal)
4734 #ifdef ENABLE_IMPORT_PARQUET
4735  } else if (boost::iequals(*p->get_name(), "parquet")) {
4736  const StringLiteral* str_literal =
4737  dynamic_cast<const StringLiteral*>(p->get_value());
4738  if (str_literal == nullptr) {
4739  throw std::runtime_error("Parquet option must be a boolean.");
4740  }
4741  if (bool_from_string_literal(str_literal)) {
4742  // not sure a parquet "table" type is proper, but to make code
4743  // look consistent in some places, let's set "table" type too
4744  copy_params.file_type = import_export::FileType::PARQUET;
4745  }
4746 #endif // ENABLE_IMPORT_PARQUET
4747  } else if (boost::iequals(*p->get_name(), "s3_access_key")) {
4748  const StringLiteral* str_literal =
4749  dynamic_cast<const StringLiteral*>(p->get_value());
4750  if (str_literal == nullptr) {
4751  throw std::runtime_error("Option s3_access_key must be a string.");
4752  }
4753  copy_params.s3_access_key = *str_literal->get_stringval();
4754  } else if (boost::iequals(*p->get_name(), "s3_secret_key")) {
4755  const StringLiteral* str_literal =
4756  dynamic_cast<const StringLiteral*>(p->get_value());
4757  if (str_literal == nullptr) {
4758  throw std::runtime_error("Option s3_secret_key must be a string.");
4759  }
4760  copy_params.s3_secret_key = *str_literal->get_stringval();
4761  } else if (boost::iequals(*p->get_name(), "s3_session_token")) {
4762  const StringLiteral* str_literal =
4763  dynamic_cast<const StringLiteral*>(p->get_value());
4764  if (str_literal == nullptr) {
4765  throw std::runtime_error("Option s3_session_token must be a string.");
4766  }
4767  copy_params.s3_session_token = *str_literal->get_stringval();
4768  } else if (boost::iequals(*p->get_name(), "s3_region")) {
4769  const StringLiteral* str_literal =
4770  dynamic_cast<const StringLiteral*>(p->get_value());
4771  if (str_literal == nullptr) {
4772  throw std::runtime_error("Option s3_region must be a string.");
4773  }
4774  copy_params.s3_region = *str_literal->get_stringval();
4775  } else if (boost::iequals(*p->get_name(), "s3_endpoint")) {
4776  const StringLiteral* str_literal =
4777  dynamic_cast<const StringLiteral*>(p->get_value());
4778  if (str_literal == nullptr) {
4779  throw std::runtime_error("Option s3_endpoint must be a string.");
4780  }
4781  copy_params.s3_endpoint = *str_literal->get_stringval();
4782  } else if (boost::iequals(*p->get_name(), "quote")) {
4783  const StringLiteral* str_literal =
4784  dynamic_cast<const StringLiteral*>(p->get_value());
4785  if (str_literal == nullptr) {
4786  throw std::runtime_error("Quote option must be a string.");
4787  } else if (str_literal->get_stringval()->length() != 1) {
4788  throw std::runtime_error("Quote must be a single character string.");
4789  }
4790  copy_params.quote = (*str_literal->get_stringval())[0];
4791  } else if (boost::iequals(*p->get_name(), "escape")) {
4792  const StringLiteral* str_literal =
4793  dynamic_cast<const StringLiteral*>(p->get_value());
4794  if (str_literal == nullptr) {
4795  throw std::runtime_error("Escape option must be a string.");
4796  } else if (str_literal->get_stringval()->length() != 1) {
4797  throw std::runtime_error("Escape must be a single character string.");
4798  }
4799  copy_params.escape = (*str_literal->get_stringval())[0];
4800  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
4801  const StringLiteral* str_literal =
4802  dynamic_cast<const StringLiteral*>(p->get_value());
4803  if (str_literal == nullptr) {
4804  throw std::runtime_error("Line_delimiter option must be a string.");
4805  } else if (str_literal->get_stringval()->length() != 1) {
4806  throw std::runtime_error("Line_delimiter must be a single character string.");
4807  }
4808  copy_params.line_delim = (*str_literal->get_stringval())[0];
4809  } else if (boost::iequals(*p->get_name(), "quoted")) {
4810  const StringLiteral* str_literal =
4811  dynamic_cast<const StringLiteral*>(p->get_value());
4812  if (str_literal == nullptr) {
4813  throw std::runtime_error("Quoted option must be a boolean.");
4814  }
4815  copy_params.quoted = bool_from_string_literal(str_literal);
4816  } else if (boost::iequals(*p->get_name(), "plain_text")) {
4817  const StringLiteral* str_literal =
4818  dynamic_cast<const StringLiteral*>(p->get_value());
4819  if (str_literal == nullptr) {
4820  throw std::runtime_error("plain_text option must be a boolean.");
4821  }
4822  copy_params.plain_text = bool_from_string_literal(str_literal);
4823  } else if (boost::iequals(*p->get_name(), "array_marker")) {
4824  const StringLiteral* str_literal =
4825  dynamic_cast<const StringLiteral*>(p->get_value());
4826  if (str_literal == nullptr) {
4827  throw std::runtime_error("Array Marker option must be a string.");
4828  } else if (str_literal->get_stringval()->length() != 2) {
4829  throw std::runtime_error(
4830  "Array Marker option must be exactly two characters. Default is {}.");
4831  }
4832  copy_params.array_begin = (*str_literal->get_stringval())[0];
4833  copy_params.array_end = (*str_literal->get_stringval())[1];
4834  } else if (boost::iequals(*p->get_name(), "array_delimiter")) {
4835  const StringLiteral* str_literal =
4836  dynamic_cast<const StringLiteral*>(p->get_value());
4837  if (str_literal == nullptr) {
4838  throw std::runtime_error("Array Delimiter option must be a string.");
4839  } else if (str_literal->get_stringval()->length() != 1) {
4840  throw std::runtime_error("Array Delimiter must be a single character string.");
4841  }
4842  copy_params.array_delim = (*str_literal->get_stringval())[0];
4843  } else if (boost::iequals(*p->get_name(), "lonlat")) {
4844  const StringLiteral* str_literal =
4845  dynamic_cast<const StringLiteral*>(p->get_value());
4846  if (str_literal == nullptr) {
4847  throw std::runtime_error("Lonlat option must be a boolean.");
4848  }
4849  copy_params.lonlat = bool_from_string_literal(str_literal);
4850  } else if (boost::iequals(*p->get_name(), "geo")) {
4851  const StringLiteral* str_literal =
4852  dynamic_cast<const StringLiteral*>(p->get_value());
4853  if (str_literal == nullptr) {
4854  throw std::runtime_error("Geo option must be a boolean.");
4855  }
4856  copy_params.file_type = bool_from_string_literal(str_literal)
4859  } else if (boost::iequals(*p->get_name(), "geo_coords_type")) {
4860  const StringLiteral* str_literal =
4861  dynamic_cast<const StringLiteral*>(p->get_value());
4862  if (str_literal == nullptr) {
4863  throw std::runtime_error("'geo_coords_type' option must be a string");
4864  }
4865  const std::string* s = str_literal->get_stringval();
4866  if (boost::iequals(*s, "geography")) {
4867  throw std::runtime_error(
4868  "GEOGRAPHY coords type not yet supported. Please use GEOMETRY.");
4869  // copy_params.geo_coords_type = kGEOGRAPHY;
4870  } else if (boost::iequals(*s, "geometry")) {
4871  copy_params.geo_coords_type = kGEOMETRY;
4872  } else {
4873  throw std::runtime_error(
4874  "Invalid string for 'geo_coords_type' option (must be 'GEOGRAPHY' or "
4875  "'GEOMETRY'): " +
4876  *s);
4877  }
4878  } else if (boost::iequals(*p->get_name(), "geo_coords_encoding")) {
4879  const StringLiteral* str_literal =
4880  dynamic_cast<const StringLiteral*>(p->get_value());
4881  if (str_literal == nullptr) {
4882  throw std::runtime_error("'geo_coords_encoding' option must be a string");
4883  }
4884  const std::string* s = str_literal->get_stringval();
4885  if (boost::iequals(*s, "none")) {
4886  copy_params.geo_coords_encoding = kENCODING_NONE;
4887  copy_params.geo_coords_comp_param = 0;
4888  } else if (boost::iequals(*s, "compressed(32)")) {
4889  copy_params.geo_coords_encoding = kENCODING_GEOINT;
4890  copy_params.geo_coords_comp_param = 32;
4891  } else {
4892  throw std::runtime_error(
4893  "Invalid string for 'geo_coords_encoding' option (must be 'NONE' or "
4894  "'COMPRESSED(32)'): " +
4895  *s);
4896  }
4897  } else if (boost::iequals(*p->get_name(), "geo_coords_srid")) {
4898  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4899  if (int_literal == nullptr) {
4900  throw std::runtime_error("'geo_coords_srid' option must be an integer");
4901  }
4902  const int srid = int_literal->get_intval();
4903  if (srid == 4326 || srid == 3857 || srid == 900913) {
4904  copy_params.geo_coords_srid = srid;
4905  } else {
4906  throw std::runtime_error(
4907  "Invalid value for 'geo_coords_srid' option (must be 4326, 3857, or "
4908  "900913): " +
4909  std::to_string(srid));
4910  }
4911  } else if (boost::iequals(*p->get_name(), "geo_layer_name")) {
4912  const StringLiteral* str_literal =
4913  dynamic_cast<const StringLiteral*>(p->get_value());
4914  if (str_literal == nullptr) {
4915  throw std::runtime_error("'geo_layer_name' option must be a string");
4916  }
4917  const std::string* layer_name = str_literal->get_stringval();
4918  if (layer_name) {
4919  copy_params.geo_layer_name = *layer_name;
4920  } else {
4921  throw std::runtime_error("Invalid value for 'geo_layer_name' option");
4922  }
4923  } else if (boost::iequals(*p->get_name(), "partitions")) {
4924  if (copy_params.file_type == import_export::FileType::POLYGON) {
4925  const auto partitions =
4926  static_cast<const StringLiteral*>(p->get_value())->get_stringval();
4927  CHECK(partitions);
4928  const auto partitions_uc = boost::to_upper_copy<std::string>(*partitions);
4929  if (partitions_uc != "REPLICATED") {
4930  throw std::runtime_error("PARTITIONS must be REPLICATED for geo COPY");
4931  }
4932  geo_copy_from_partitions_ = partitions_uc;
4933  } else {
4934  throw std::runtime_error("PARTITIONS option not supported for non-geo COPY: " +
4935  *p->get_name());
4936  }
4937  } else if (boost::iequals(*p->get_name(), "geo_assign_render_groups")) {
4938  const StringLiteral* str_literal =
4939  dynamic_cast<const StringLiteral*>(p->get_value());
4940  if (str_literal == nullptr) {
4941  throw std::runtime_error("geo_assign_render_groups option must be a boolean.");
4942  }
4943  copy_params.geo_assign_render_groups = bool_from_string_literal(str_literal);
4944  } else if (boost::iequals(*p->get_name(), "geo_explode_collections")) {
4945  const StringLiteral* str_literal =
4946  dynamic_cast<const StringLiteral*>(p->get_value());
4947  if (str_literal == nullptr) {
4948  throw std::runtime_error("geo_explode_collections option must be a boolean.");
4949  }
4950  copy_params.geo_explode_collections = bool_from_string_literal(str_literal);
4951  } else if (boost::iequals(*p->get_name(), "source_srid")) {
4952  const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
4953  if (int_literal == nullptr) {
4954  throw std::runtime_error("'source_srid' option must be an integer");
4955  }
4956  const int srid = int_literal->get_intval();
4957  if (copy_params.file_type == import_export::FileType::DELIMITED) {
4958  copy_params.source_srid = srid;
4959  } else {
4960  throw std::runtime_error(
4961  "'source_srid' option can only be used on csv/tsv files");
4962  }
4963  } else if (boost::iequals(*p->get_name(), "regex_path_filter")) {
4964  const StringLiteral* str_literal =
4965  dynamic_cast<const StringLiteral*>(p->get_value());
4966  if (str_literal == nullptr) {
4967  throw std::runtime_error("Option regex_path_filter must be a string.");
4968  }
4969  const auto string_val = *str_literal->get_stringval();
4970  copy_params.regex_path_filter =
4971  string_val.empty() ? std::nullopt : std::optional<std::string>{string_val};
4972  } else {
4973  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
4974  }
4975  }
4976  }
4977 
4978  std::string tr;
4979  if (copy_params.file_type == import_export::FileType::POLYGON) {
4980  // geo import
4981  // we do nothing here, except stash the parameters so we can
4982  // do the import when we unwind to the top of the handler
4983  geo_copy_from_file_name_ = file_path;
4984  geo_copy_from_copy_params_ = copy_params;
4985  was_geo_copy_from_ = true;
4986 
4987  // the result string
4988  // @TODO simon.eves put something more useful in here
4989  // except we really can't because we haven't done the import yet!
4990  if (td) {
4991  tr = std::string("Appending geo to table '") + *table_ + std::string("'...");
4992  } else {
4993  tr = std::string("Creating table '") + *table_ +
4994  std::string("' and importing geo...");
4995  }
4996  } else {
4997  if (td) {
4998  CHECK(td_with_lock);
4999 
5000  // regular import
5001  auto importer = importer_factory(catalog, td, file_path, copy_params);
5002  auto start_time = ::toString(std::chrono::system_clock::now());
5003  auto prev_table_epoch = importer->getLoader()->getTableEpochs();
5005  auto query_session = session.get_session_id();
5006  auto query_str = "COPYING " + td->tableName;
5008  executor->enrollQuerySession(query_session,
5009  query_str,
5010  start_time,
5012  QuerySessionStatus::QueryStatus::RUNNING_IMPORTER);
5013  }
5014 
5015  ScopeGuard clearInterruptStatus =
5016  [executor, &query_str, &query_session, &start_time, &importer] {
5017  // reset the runtime query interrupt status
5019  executor->clearQuerySessionStatus(query_session, start_time);
5020  }
5021  };
5022  import_export::ImportStatus import_result;
5023  auto ms =
5024  measure<>::execution([&]() { import_result = importer->import(&session); });
5025  total_time += ms;
5026  // results
5027  if (!import_result.load_failed &&
5028  import_result.rows_rejected > copy_params.max_reject) {
5029  LOG(ERROR) << "COPY exited early due to reject records count during multi file "
5030  "processing ";
5031  // if we have crossed the truncated load threshold
5032  import_result.load_failed = true;
5033  import_result.load_msg =
5034  "COPY exited early due to reject records count during multi file "
5035  "processing ";
5036  success_ = false;
5037  }
5038  if (!import_result.load_failed) {
5039  tr = std::string(
5040  "Loaded: " + std::to_string(import_result.rows_completed) +
5041  " recs, Rejected: " + std::to_string(import_result.rows_rejected) +
5042  " recs in " + std::to_string((double)total_time / 1000.0) + " secs");
5043  } else {
5044  tr = std::string("Loader Failed due to : " + import_result.load_msg + " in " +
5045  std::to_string((double)total_time / 1000.0) + " secs");
5046  }
5047  } else {
5048  throw std::runtime_error("Table '" + *table_ + "' must exist before COPY FROM");
5049  }
5050  }
5051 
5052  return_message.reset(new std::string(tr));
5053  LOG(INFO) << tr;
5054 } // namespace Parser
std::string s3_secret_key
Definition: CopyParams.h:66
std::string toString(const ExtArgumentType &sig_type)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:203
std::unique_ptr< std::string > return_message
Definition: ParserNode.h:1484
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:155
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:163
import_export::CopyParams geo_copy_from_copy_params_
Definition: ParserNode.h:1514
std::string geo_copy_from_partitions_
Definition: ParserNode.h:1515
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:119
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
Definition: ParserNode.cpp:945
std::string to_string(char const *&&v)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:171
ImportHeaderRow has_header
Definition: CopyParams.h:51
std::optional< std::string > regex_path_filter
Definition: CopyParams.h:86
std::string geo_copy_from_file_name_
Definition: ParserNode.h:1513
std::unique_ptr< std::string > file_pattern_
Definition: ParserNode.h:1508
std::unique_ptr< std::string > table_
Definition: ParserNode.h:1507
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:729
std::string get_session_id() const
Definition: SessionInfo.h:78
std::string geo_layer_name
Definition: CopyParams.h:82
Catalog & getCatalog() const
Definition: SessionInfo.h:67
std::string s3_session_token
Definition: CopyParams.h:67
static std::shared_ptr< MutexType > getMutex(const LockType lockType, const KeyType &key)
Definition: LegacyLockMgr.h:51
#define CHECK(condition)
Definition: Logger.h:209
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1510
std::string userLoggable() const
Definition: SysCatalog.cpp:126
std::string s3_access_key
Definition: CopyParams.h:65
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:73
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:384
EncodingType geo_coords_encoding
Definition: CopyParams.h:77

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Parser::CopyTableStmt::get_geo_copy_from_payload ( std::string &  geo_copy_from_table,
std::string &  geo_copy_from_file_name,
import_export::CopyParams geo_copy_from_copy_params,
std::string &  geo_copy_from_partitions 
)
inline

Definition at line 1495 of file ParserNode.h.

References geo_copy_from_copy_params_, geo_copy_from_file_name_, geo_copy_from_partitions_, table_, and was_geo_copy_from_.

1498  {
1499  geo_copy_from_table = *table_;
1500  geo_copy_from_file_name = geo_copy_from_file_name_;
1501  geo_copy_from_copy_params = geo_copy_from_copy_params_;
1502  geo_copy_from_partitions = geo_copy_from_partitions_;
1503  was_geo_copy_from_ = false;
1504  }
import_export::CopyParams geo_copy_from_copy_params_
Definition: ParserNode.h:1514
std::string geo_copy_from_partitions_
Definition: ParserNode.h:1515
std::string geo_copy_from_file_name_
Definition: ParserNode.h:1513
std::unique_ptr< std::string > table_
Definition: ParserNode.h:1507
bool Parser::CopyTableStmt::get_success ( ) const
inline

Definition at line 1491 of file ParserNode.h.

References success_.

1491 { return success_; }
std::string& Parser::CopyTableStmt::get_table ( ) const
inline

Definition at line 1486 of file ParserNode.h.

References CHECK, and table_.

1486  {
1487  CHECK(table_);
1488  return *table_;
1489  }
std::unique_ptr< std::string > table_
Definition: ParserNode.h:1507
#define CHECK(condition)
Definition: Logger.h:209
bool Parser::CopyTableStmt::was_geo_copy_from ( ) const
inline

Definition at line 1493 of file ParserNode.h.

References was_geo_copy_from_.

1493 { return was_geo_copy_from_; }

Member Data Documentation

std::unique_ptr<std::string> Parser::CopyTableStmt::file_pattern_
private

Definition at line 1508 of file ParserNode.h.

Referenced by CopyTableStmt(), and execute().

import_export::CopyParams Parser::CopyTableStmt::geo_copy_from_copy_params_
private

Definition at line 1514 of file ParserNode.h.

Referenced by execute(), and get_geo_copy_from_payload().

std::string Parser::CopyTableStmt::geo_copy_from_file_name_
private

Definition at line 1513 of file ParserNode.h.

Referenced by execute(), and get_geo_copy_from_payload().

std::string Parser::CopyTableStmt::geo_copy_from_partitions_
private

Definition at line 1515 of file ParserNode.h.

Referenced by execute(), and get_geo_copy_from_payload().

std::list<std::unique_ptr<NameValueAssign> > Parser::CopyTableStmt::options_
private

Definition at line 1510 of file ParserNode.h.

Referenced by CopyTableStmt(), and execute().

std::unique_ptr<std::string> Parser::CopyTableStmt::return_message

Definition at line 1484 of file ParserNode.h.

Referenced by execute().

bool Parser::CopyTableStmt::success_
private

Definition at line 1509 of file ParserNode.h.

Referenced by execute(), and get_success().

std::unique_ptr<std::string> Parser::CopyTableStmt::table_
private

Definition at line 1507 of file ParserNode.h.

Referenced by CopyTableStmt(), execute(), get_geo_copy_from_payload(), and get_table().

bool Parser::CopyTableStmt::was_geo_copy_from_ = false
private

Definition at line 1512 of file ParserNode.h.

Referenced by execute(), get_geo_copy_from_payload(), and was_geo_copy_from().


The documentation for this class was generated from the following files: