OmniSciDB  f632821e96
Parser::ExportQueryStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::ExportQueryStmt:
+ Collaboration diagram for Parser::ExportQueryStmt:

Public Member Functions

 ExportQueryStmt (std::string *q, std::string *p, std::list< NameValueAssign *> *o)
 
void execute (const Catalog_Namespace::SessionInfo &session) override
 
const std::string get_select_stmt () const
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

DistributedConnectorleafs_connector_ = nullptr
 

Private Member Functions

void parseOptions (import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
 

Private Attributes

std::unique_ptr< std::string > select_stmt
 
std::unique_ptr< std::string > file_path
 
std::list< std::unique_ptr< NameValueAssign > > options
 

Detailed Description

Definition at line 1806 of file ParserNode.h.

Constructor & Destructor Documentation

◆ ExportQueryStmt()

Parser::ExportQueryStmt::ExportQueryStmt ( std::string *  q,
std::string *  p,
std::list< NameValueAssign *> *  o 
)
inline

Definition at line 1808 of file ParserNode.h.

1809  : select_stmt(q), file_path(p) {
1810  if (o) {
1811  for (const auto e : *o) {
1812  options.emplace_back(e);
1813  }
1814  delete o;
1815  }
1816  }
std::unique_ptr< std::string > file_path
Definition: ParserNode.h:1824
std::list< std::unique_ptr< NameValueAssign > > options
Definition: ParserNode.h:1825
std::unique_ptr< std::string > select_stmt
Definition: ParserNode.h:1823

Member Function Documentation

◆ execute()

void Parser::ExportQueryStmt::execute ( const Catalog_Namespace::SessionInfo session)
overridevirtual

Implements Parser::DDLStmt.

Definition at line 4353 of file ParserNode.cpp.

References import_export::QueryExporter::create(), query_state::QueryState::create(), ddl_utils::EXPORT, Catalog_Namespace::SessionInfo::get_session_id(), Catalog_Namespace::SessionInfo::getCatalog(), import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kNone, Parser::LocalConnector::query(), STDLOG, and ddl_utils::validate_allowed_file_path().

4353  {
4354  auto session_copy = session;
4355  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4356  &session_copy, boost::null_deleter());
4358  auto stdlog = STDLOG(query_state);
4359  auto query_state_proxy = query_state->createQueryStateProxy();
4360 
4361  auto& catalog = session.getCatalog();
4362 
4363  LocalConnector local_connector;
4364 
4365  if (!leafs_connector_) {
4366  leafs_connector_ = &local_connector;
4367  }
4368 
4369  import_export::CopyParams copy_params;
4370  // @TODO(se) move rest to CopyParams when we have a Thrift endpoint
4373  std::string layer_name;
4378 
4379  parseOptions(copy_params, file_type, layer_name, file_compression, array_null_handling);
4380 
4381  if (file_path->empty()) {
4382  throw std::runtime_error("Invalid file path for COPY TO");
4383  } else if (!boost::filesystem::path(*file_path).is_absolute()) {
4384  std::string file_name = boost::filesystem::path(*file_path).filename().string();
4385  std::string file_dir =
4386  catalog.getBasePath() + "/mapd_export/" + session.get_session_id() + "/";
4387  if (!boost::filesystem::exists(file_dir)) {
4388  if (!boost::filesystem::create_directories(file_dir)) {
4389  throw std::runtime_error("Directory " + file_dir + " cannot be created.");
4390  }
4391  }
4392  *file_path = file_dir + file_name;
4393  } else {
4394  // Above branch will create a new file in the mapd_export directory. If that path is
4395  // not exercised, go through applicable file path validations.
4398  }
4399 
4400  // get column info
4401  auto column_info_result =
4402  local_connector.query(query_state_proxy, *select_stmt, {}, true);
4403 
4404  // create exporter for requested file type
4405  auto query_exporter = import_export::QueryExporter::create(file_type);
4406 
4407  // default layer name to file path stem if it wasn't specified
4408  if (layer_name.size() == 0) {
4409  layer_name = boost::filesystem::path(*file_path).stem().string();
4410  }
4411 
4412  // begin export
4413  query_exporter->beginExport(*file_path,
4414  layer_name,
4415  copy_params,
4416  column_info_result.targets_meta,
4417  file_compression,
4418  array_null_handling);
4419 
4420  // how many fragments?
4421  size_t outer_frag_count =
4422  leafs_connector_->getOuterFragmentCount(query_state_proxy, *select_stmt);
4423  size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
4424 
4425  // loop fragments
4426  for (size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
4427  // limit the query to just this fragment
4428  std::vector<size_t> allowed_outer_fragment_indices;
4429  if (outer_frag_count) {
4430  allowed_outer_fragment_indices.push_back(outer_frag_idx);
4431  }
4432 
4433  // run the query
4434  std::vector<AggregatedResult> query_results = leafs_connector_->query(
4435  query_state_proxy, *select_stmt, allowed_outer_fragment_indices);
4436 
4437  // export the results
4438  query_exporter->exportResults(query_results);
4439  }
4440 
4441  // end export
4442  query_exporter->endExport();
4443 }
virtual std::vector< AggregatedResult > query(QueryStateProxy, std::string &sql_query_string, std::vector< size_t > outer_frag_indices)=0
void parseOptions(import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
DistributedConnector * leafs_connector_
Definition: ParserNode.h:1820
std::string get_session_id() const
Definition: SessionInfo.h:73
static std::unique_ptr< QueryExporter > create(const FileType file_type)
std::unique_ptr< std::string > file_path
Definition: ParserNode.h:1824
Catalog & getCatalog() const
Definition: SessionInfo.h:65
static std::shared_ptr< QueryState > create(ARGS &&... args)
Definition: QueryState.h:140
virtual size_t getOuterFragmentCount(QueryStateProxy, std::string &sql_query_string)=0
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:611
std::unique_ptr< std::string > select_stmt
Definition: ParserNode.h:1823
#define STDLOG(...)
Definition: QueryState.h:225
+ Here is the call graph for this function:

◆ get_select_stmt()

const std::string Parser::ExportQueryStmt::get_select_stmt ( ) const
inline

Definition at line 1818 of file ParserNode.h.

Referenced by DBHandler::sql_execute_impl().

1818 { return *select_stmt; }
std::unique_ptr< std::string > select_stmt
Definition: ParserNode.h:1823
+ Here is the caller graph for this function:

◆ parseOptions()

void Parser::ExportQueryStmt::parseOptions ( import_export::CopyParams copy_params,
import_export::QueryExporter::FileType file_type,
std::string &  layer_name,
import_export::QueryExporter::FileCompression file_compression,
import_export::QueryExporter::ArrayNullHandling array_null_handling 
)
private

Definition at line 4445 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::bool_from_string_literal(), import_export::CopyParams::delimiter, import_export::CopyParams::escape, Parser::StringLiteral::get_stringval(), import_export::HAS_HEADER, import_export::CopyParams::has_header, import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kExportSentinels, import_export::QueryExporter::kExportZeros, import_export::QueryExporter::kGeoJSON, import_export::QueryExporter::kGeoJSONL, import_export::QueryExporter::kGZip, import_export::QueryExporter::kNone, import_export::QueryExporter::kNullEntireField, import_export::QueryExporter::kShapefile, import_export::QueryExporter::kZip, import_export::CopyParams::line_delim, import_export::NO_HEADER, import_export::CopyParams::null_str, import_export::CopyParams::quote, and import_export::CopyParams::quoted.

4450  {
4451  // defaults for non-CopyParams values
4453  layer_name.clear();
4455 
4456  if (!options.empty()) {
4457  for (auto& p : options) {
4458  if (boost::iequals(*p->get_name(), "delimiter")) {
4459  const StringLiteral* str_literal =
4460  dynamic_cast<const StringLiteral*>(p->get_value());
4461  if (str_literal == nullptr) {
4462  throw std::runtime_error("Delimiter option must be a string.");
4463  } else if (str_literal->get_stringval()->length() != 1) {
4464  throw std::runtime_error("Delimiter must be a single character string.");
4465  }
4466  copy_params.delimiter = (*str_literal->get_stringval())[0];
4467  } else if (boost::iequals(*p->get_name(), "nulls")) {
4468  const StringLiteral* str_literal =
4469  dynamic_cast<const StringLiteral*>(p->get_value());
4470  if (str_literal == nullptr) {
4471  throw std::runtime_error("Nulls option must be a string.");
4472  }
4473  copy_params.null_str = *str_literal->get_stringval();
4474  } else if (boost::iequals(*p->get_name(), "header")) {
4475  const StringLiteral* str_literal =
4476  dynamic_cast<const StringLiteral*>(p->get_value());
4477  if (str_literal == nullptr) {
4478  throw std::runtime_error("Header option must be a boolean.");
4479  }
4480  copy_params.has_header = bool_from_string_literal(str_literal)
4483  } else if (boost::iequals(*p->get_name(), "quote")) {
4484  const StringLiteral* str_literal =
4485  dynamic_cast<const StringLiteral*>(p->get_value());
4486  if (str_literal == nullptr) {
4487  throw std::runtime_error("Quote option must be a string.");
4488  } else if (str_literal->get_stringval()->length() != 1) {
4489  throw std::runtime_error("Quote must be a single character string.");
4490  }
4491  copy_params.quote = (*str_literal->get_stringval())[0];
4492  } else if (boost::iequals(*p->get_name(), "escape")) {
4493  const StringLiteral* str_literal =
4494  dynamic_cast<const StringLiteral*>(p->get_value());
4495  if (str_literal == nullptr) {
4496  throw std::runtime_error("Escape option must be a string.");
4497  } else if (str_literal->get_stringval()->length() != 1) {
4498  throw std::runtime_error("Escape must be a single character string.");
4499  }
4500  copy_params.escape = (*str_literal->get_stringval())[0];
4501  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
4502  const StringLiteral* str_literal =
4503  dynamic_cast<const StringLiteral*>(p->get_value());
4504  if (str_literal == nullptr) {
4505  throw std::runtime_error("Line_delimiter option must be a string.");
4506  } else if (str_literal->get_stringval()->length() != 1) {
4507  throw std::runtime_error("Line_delimiter must be a single character string.");
4508  }
4509  copy_params.line_delim = (*str_literal->get_stringval())[0];
4510  } else if (boost::iequals(*p->get_name(), "quoted")) {
4511  const StringLiteral* str_literal =
4512  dynamic_cast<const StringLiteral*>(p->get_value());
4513  if (str_literal == nullptr) {
4514  throw std::runtime_error("Quoted option must be a boolean.");
4515  }
4516  copy_params.quoted = bool_from_string_literal(str_literal);
4517  } else if (boost::iequals(*p->get_name(), "file_type")) {
4518  const StringLiteral* str_literal =
4519  dynamic_cast<const StringLiteral*>(p->get_value());
4520  if (str_literal == nullptr) {
4521  throw std::runtime_error("File Type option must be a string.");
4522  }
4523  auto file_type_str =
4524  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
4525  if (file_type_str == "csv") {
4527  } else if (file_type_str == "geojson") {
4529  } else if (file_type_str == "geojsonl") {
4531  } else if (file_type_str == "shapefile") {
4533  } else {
4534  throw std::runtime_error(
4535  "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL' or 'Shapefile'");
4536  }
4537  } else if (boost::iequals(*p->get_name(), "layer_name")) {
4538  const StringLiteral* str_literal =
4539  dynamic_cast<const StringLiteral*>(p->get_value());
4540  if (str_literal == nullptr) {
4541  throw std::runtime_error("Layer Name option must be a string.");
4542  }
4543  layer_name = *str_literal->get_stringval();
4544  } else if (boost::iequals(*p->get_name(), "file_compression")) {
4545  const StringLiteral* str_literal =
4546  dynamic_cast<const StringLiteral*>(p->get_value());
4547  if (str_literal == nullptr) {
4548  throw std::runtime_error("File Compression option must be a string.");
4549  }
4550  auto file_compression_str =
4551  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
4552  if (file_compression_str == "none") {
4554  } else if (file_compression_str == "gzip") {
4556  } else if (file_compression_str == "zip") {
4558  } else {
4559  throw std::runtime_error(
4560  "File Compression option must be 'None', 'GZip', or 'Zip'");
4561  }
4562  } else if (boost::iequals(*p->get_name(), "array_null_handling")) {
4563  const StringLiteral* str_literal =
4564  dynamic_cast<const StringLiteral*>(p->get_value());
4565  if (str_literal == nullptr) {
4566  throw std::runtime_error("Array Null Handling option must be a string.");
4567  }
4568  auto array_null_handling_str =
4569  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
4570  if (array_null_handling_str == "abort") {
4571  array_null_handling =
4573  } else if (array_null_handling_str == "raw") {
4574  array_null_handling =
4576  } else if (array_null_handling_str == "zero") {
4577  array_null_handling =
4579  } else if (array_null_handling_str == "nullfield") {
4580  array_null_handling =
4582  } else {
4583  throw std::runtime_error(
4584  "Array Null Handling option must be 'Abort', 'Raw', 'Zero', or "
4585  "'NullField'");
4586  }
4587  } else {
4588  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
4589  }
4590  }
4591  }
4592 }
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
Definition: ParserNode.cpp:907
ImportHeaderRow has_header
Definition: CopyParams.h:48
std::list< std::unique_ptr< NameValueAssign > > options
Definition: ParserNode.h:1825
+ Here is the call graph for this function:

Member Data Documentation

◆ file_path

std::unique_ptr<std::string> Parser::ExportQueryStmt::file_path
private

Definition at line 1824 of file ParserNode.h.

◆ leafs_connector_

DistributedConnector* Parser::ExportQueryStmt::leafs_connector_ = nullptr

Definition at line 1820 of file ParserNode.h.

◆ options

std::list<std::unique_ptr<NameValueAssign> > Parser::ExportQueryStmt::options
private

Definition at line 1825 of file ParserNode.h.

◆ select_stmt

std::unique_ptr<std::string> Parser::ExportQueryStmt::select_stmt
private

Definition at line 1823 of file ParserNode.h.


The documentation for this class was generated from the following files: