OmniSciDB  ba1bac9284
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Parser::ExportQueryStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::ExportQueryStmt:
+ Collaboration diagram for Parser::ExportQueryStmt:

Public Member Functions

 ExportQueryStmt (std::string *q, std::string *p, std::list< NameValueAssign * > *o)
 
void execute (const Catalog_Namespace::SessionInfo &session) override
 
const std::string get_select_stmt () const
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

DistributedConnectorleafs_connector_ = nullptr
 

Private Member Functions

void parseOptions (import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
 

Private Attributes

std::unique_ptr< std::string > select_stmt
 
std::unique_ptr< std::string > file_path
 
std::list< std::unique_ptr
< NameValueAssign > > 
options
 

Detailed Description

Definition at line 1860 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::ExportQueryStmt::ExportQueryStmt ( std::string *  q,
std::string *  p,
std::list< NameValueAssign * > *  o 
)
inline

Definition at line 1862 of file ParserNode.h.

References options.

1863  : select_stmt(q), file_path(p) {
1864  if (o) {
1865  for (const auto e : *o) {
1866  options.emplace_back(e);
1867  }
1868  delete o;
1869  }
1870  }
std::unique_ptr< std::string > file_path
Definition: ParserNode.h:1878
std::list< std::unique_ptr< NameValueAssign > > options
Definition: ParserNode.h:1879
std::unique_ptr< std::string > select_stmt
Definition: ParserNode.h:1877

Member Function Documentation

void Parser::ExportQueryStmt::execute ( const Catalog_Namespace::SessionInfo session)
overridevirtual

Implements Parser::DDLStmt.

Definition at line 5269 of file ParserNode.cpp.

References import_export::QueryExporter::create(), query_state::QueryState::create(), ddl_utils::EXPORT, g_base_path, Catalog_Namespace::SessionInfo::get_session_id(), import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kNone, Parser::LocalConnector::query(), STDLOG, and ddl_utils::validate_allowed_file_path().

Referenced by omnisci.cursor.Cursor::executemany().

5269  {
5270  auto session_copy = session;
5271  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
5272  &session_copy, boost::null_deleter());
5273  auto query_state = query_state::QueryState::create(session_ptr, *select_stmt);
5274  auto stdlog = STDLOG(query_state);
5275  auto query_state_proxy = query_state->createQueryStateProxy();
5276 
5277  LocalConnector local_connector;
5278 
5279  if (!leafs_connector_) {
5280  leafs_connector_ = &local_connector;
5281  }
5282 
5283  import_export::CopyParams copy_params;
5284  // @TODO(se) move rest to CopyParams when we have a Thrift endpoint
5287  std::string layer_name;
5292 
5293  parseOptions(copy_params, file_type, layer_name, file_compression, array_null_handling);
5294 
5295  if (file_path->empty()) {
5296  throw std::runtime_error("Invalid file path for COPY TO");
5297  } else if (!boost::filesystem::path(*file_path).is_absolute()) {
5298  std::string file_name = boost::filesystem::path(*file_path).filename().string();
5299  std::string file_dir = g_base_path + "/mapd_export/" + session.get_session_id() + "/";
5300  if (!boost::filesystem::exists(file_dir)) {
5301  if (!boost::filesystem::create_directories(file_dir)) {
5302  throw std::runtime_error("Directory " + file_dir + " cannot be created.");
5303  }
5304  }
5305  *file_path = file_dir + file_name;
5306  } else {
5307  // Above branch will create a new file in the mapd_export directory. If that path
5308  // is not exercised, go through applicable file path validations.
5311  }
5312 
5313  // get column info
5314  auto column_info_result =
5315  local_connector.query(query_state_proxy, *select_stmt, {}, true, false);
5316 
5317  // create exporter for requested file type
5318  auto query_exporter = import_export::QueryExporter::create(file_type);
5319 
5320  // default layer name to file path stem if it wasn't specified
5321  if (layer_name.size() == 0) {
5322  layer_name = boost::filesystem::path(*file_path).stem().string();
5323  }
5324 
5325  // begin export
5326  query_exporter->beginExport(*file_path,
5327  layer_name,
5328  copy_params,
5329  column_info_result.targets_meta,
5330  file_compression,
5331  array_null_handling);
5332 
5333  // how many fragments?
5334  size_t outer_frag_count =
5335  leafs_connector_->getOuterFragmentCount(query_state_proxy, *select_stmt);
5336  size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
5337 
5338  // loop fragments
5339  for (size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
5340  // limit the query to just this fragment
5341  std::vector<size_t> allowed_outer_fragment_indices;
5342  if (outer_frag_count) {
5343  allowed_outer_fragment_indices.push_back(outer_frag_idx);
5344  }
5345 
5346  // run the query
5347  std::vector<AggregatedResult> query_results = leafs_connector_->query(
5348  query_state_proxy, *select_stmt, allowed_outer_fragment_indices, false);
5349 
5350  // export the results
5351  query_exporter->exportResults(query_results);
5352  }
5353 
5354  // end export
5355  query_exporter->endExport();
5356 }
void parseOptions(import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
static std::shared_ptr< QueryState > create(ARGS &&...args)
Definition: QueryState.h:141
DistributedConnector * leafs_connector_
Definition: ParserNode.h:1874
static std::unique_ptr< QueryExporter > create(const FileType file_type)
std::unique_ptr< std::string > file_path
Definition: ParserNode.h:1878
std::string g_base_path
Definition: SysCatalog.cpp:62
virtual size_t getOuterFragmentCount(QueryStateProxy, std::string &sql_query_string)=0
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:634
std::string get_session_id() const
Definition: SessionInfo.h:78
virtual std::vector< AggregatedResult > query(QueryStateProxy, std::string &sql_query_string, std::vector< size_t > outer_frag_indices, bool allow_interrupt)=0
std::unique_ptr< std::string > select_stmt
Definition: ParserNode.h:1877
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::string Parser::ExportQueryStmt::get_select_stmt ( ) const
inline

Definition at line 1872 of file ParserNode.h.

References select_stmt.

Referenced by DBHandler::sql_execute_impl().

1872 { return *select_stmt; }
std::unique_ptr< std::string > select_stmt
Definition: ParserNode.h:1877

+ Here is the caller graph for this function:

void Parser::ExportQueryStmt::parseOptions ( import_export::CopyParams copy_params,
import_export::QueryExporter::FileType file_type,
std::string &  layer_name,
import_export::QueryExporter::FileCompression file_compression,
import_export::QueryExporter::ArrayNullHandling array_null_handling 
)
private

Definition at line 5358 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::bool_from_string_literal(), import_export::CopyParams::delimiter, import_export::CopyParams::escape, Parser::StringLiteral::get_stringval(), import_export::HAS_HEADER, import_export::CopyParams::has_header, import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kExportSentinels, import_export::QueryExporter::kExportZeros, import_export::QueryExporter::kGeoJSON, import_export::QueryExporter::kGeoJSONL, import_export::QueryExporter::kGZip, import_export::QueryExporter::kNone, import_export::QueryExporter::kNullEntireField, import_export::QueryExporter::kShapefile, import_export::QueryExporter::kZip, import_export::CopyParams::line_delim, import_export::NO_HEADER, import_export::CopyParams::null_str, import_export::CopyParams::quote, and import_export::CopyParams::quoted.

5363  {
5364  // defaults for non-CopyParams values
5366  layer_name.clear();
5368 
5369  if (!options.empty()) {
5370  for (auto& p : options) {
5371  if (boost::iequals(*p->get_name(), "delimiter")) {
5372  const StringLiteral* str_literal =
5373  dynamic_cast<const StringLiteral*>(p->get_value());
5374  if (str_literal == nullptr) {
5375  throw std::runtime_error("Delimiter option must be a string.");
5376  } else if (str_literal->get_stringval()->length() != 1) {
5377  throw std::runtime_error("Delimiter must be a single character string.");
5378  }
5379  copy_params.delimiter = (*str_literal->get_stringval())[0];
5380  } else if (boost::iequals(*p->get_name(), "nulls")) {
5381  const StringLiteral* str_literal =
5382  dynamic_cast<const StringLiteral*>(p->get_value());
5383  if (str_literal == nullptr) {
5384  throw std::runtime_error("Nulls option must be a string.");
5385  }
5386  copy_params.null_str = *str_literal->get_stringval();
5387  } else if (boost::iequals(*p->get_name(), "header")) {
5388  const StringLiteral* str_literal =
5389  dynamic_cast<const StringLiteral*>(p->get_value());
5390  if (str_literal == nullptr) {
5391  throw std::runtime_error("Header option must be a boolean.");
5392  }
5393  copy_params.has_header = bool_from_string_literal(str_literal)
5396  } else if (boost::iequals(*p->get_name(), "quote")) {
5397  const StringLiteral* str_literal =
5398  dynamic_cast<const StringLiteral*>(p->get_value());
5399  if (str_literal == nullptr) {
5400  throw std::runtime_error("Quote option must be a string.");
5401  } else if (str_literal->get_stringval()->length() != 1) {
5402  throw std::runtime_error("Quote must be a single character string.");
5403  }
5404  copy_params.quote = (*str_literal->get_stringval())[0];
5405  } else if (boost::iequals(*p->get_name(), "escape")) {
5406  const StringLiteral* str_literal =
5407  dynamic_cast<const StringLiteral*>(p->get_value());
5408  if (str_literal == nullptr) {
5409  throw std::runtime_error("Escape option must be a string.");
5410  } else if (str_literal->get_stringval()->length() != 1) {
5411  throw std::runtime_error("Escape must be a single character string.");
5412  }
5413  copy_params.escape = (*str_literal->get_stringval())[0];
5414  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
5415  const StringLiteral* str_literal =
5416  dynamic_cast<const StringLiteral*>(p->get_value());
5417  if (str_literal == nullptr) {
5418  throw std::runtime_error("Line_delimiter option must be a string.");
5419  } else if (str_literal->get_stringval()->length() != 1) {
5420  throw std::runtime_error("Line_delimiter must be a single character string.");
5421  }
5422  copy_params.line_delim = (*str_literal->get_stringval())[0];
5423  } else if (boost::iequals(*p->get_name(), "quoted")) {
5424  const StringLiteral* str_literal =
5425  dynamic_cast<const StringLiteral*>(p->get_value());
5426  if (str_literal == nullptr) {
5427  throw std::runtime_error("Quoted option must be a boolean.");
5428  }
5429  copy_params.quoted = bool_from_string_literal(str_literal);
5430  } else if (boost::iequals(*p->get_name(), "file_type")) {
5431  const StringLiteral* str_literal =
5432  dynamic_cast<const StringLiteral*>(p->get_value());
5433  if (str_literal == nullptr) {
5434  throw std::runtime_error("File Type option must be a string.");
5435  }
5436  auto file_type_str =
5437  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
5438  if (file_type_str == "csv") {
5440  } else if (file_type_str == "geojson") {
5442  } else if (file_type_str == "geojsonl") {
5444  } else if (file_type_str == "shapefile") {
5446  } else {
5447  throw std::runtime_error(
5448  "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL' or 'Shapefile'");
5449  }
5450  } else if (boost::iequals(*p->get_name(), "layer_name")) {
5451  const StringLiteral* str_literal =
5452  dynamic_cast<const StringLiteral*>(p->get_value());
5453  if (str_literal == nullptr) {
5454  throw std::runtime_error("Layer Name option must be a string.");
5455  }
5456  layer_name = *str_literal->get_stringval();
5457  } else if (boost::iequals(*p->get_name(), "file_compression")) {
5458  const StringLiteral* str_literal =
5459  dynamic_cast<const StringLiteral*>(p->get_value());
5460  if (str_literal == nullptr) {
5461  throw std::runtime_error("File Compression option must be a string.");
5462  }
5463  auto file_compression_str =
5464  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
5465  if (file_compression_str == "none") {
5467  } else if (file_compression_str == "gzip") {
5469  } else if (file_compression_str == "zip") {
5471  } else {
5472  throw std::runtime_error(
5473  "File Compression option must be 'None', 'GZip', or 'Zip'");
5474  }
5475  } else if (boost::iequals(*p->get_name(), "array_null_handling")) {
5476  const StringLiteral* str_literal =
5477  dynamic_cast<const StringLiteral*>(p->get_value());
5478  if (str_literal == nullptr) {
5479  throw std::runtime_error("Array Null Handling option must be a string.");
5480  }
5481  auto array_null_handling_str =
5482  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
5483  if (array_null_handling_str == "abort") {
5484  array_null_handling =
5486  } else if (array_null_handling_str == "raw") {
5487  array_null_handling =
5489  } else if (array_null_handling_str == "zero") {
5490  array_null_handling =
5492  } else if (array_null_handling_str == "nullfield") {
5493  array_null_handling =
5495  } else {
5496  throw std::runtime_error(
5497  "Array Null Handling option must be 'Abort', 'Raw', 'Zero', or "
5498  "'NullField'");
5499  }
5500  } else {
5501  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
5502  }
5503  }
5504  }
5505 }
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
Definition: ParserNode.cpp:921
ImportHeaderRow has_header
Definition: CopyParams.h:48
std::list< std::unique_ptr< NameValueAssign > > options
Definition: ParserNode.h:1879

+ Here is the call graph for this function:

Member Data Documentation

std::unique_ptr<std::string> Parser::ExportQueryStmt::file_path
private

Definition at line 1878 of file ParserNode.h.

DistributedConnector* Parser::ExportQueryStmt::leafs_connector_ = nullptr

Definition at line 1874 of file ParserNode.h.

std::list<std::unique_ptr<NameValueAssign> > Parser::ExportQueryStmt::options
private

Definition at line 1879 of file ParserNode.h.

Referenced by ExportQueryStmt().

std::unique_ptr<std::string> Parser::ExportQueryStmt::select_stmt
private

Definition at line 1877 of file ParserNode.h.

Referenced by get_select_stmt().


The documentation for this class was generated from the following files: