OmniSciDB  eb3a3d0a03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Parser::ExportQueryStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::ExportQueryStmt:
+ Collaboration diagram for Parser::ExportQueryStmt:

Public Member Functions

 ExportQueryStmt (std::string *q, std::string *p, std::list< NameValueAssign * > *o)
 
 ExportQueryStmt (const rapidjson::Value &payload)
 
void execute (const Catalog_Namespace::SessionInfo &session) override
 
const std::string get_select_stmt () const
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

DistributedConnectorleafs_connector_ = nullptr
 

Private Member Functions

void parseOptions (import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
 

Private Attributes

std::unique_ptr< std::string > select_stmt_
 
std::unique_ptr< std::string > file_path_
 
std::list< std::unique_ptr
< NameValueAssign > > 
options_
 

Detailed Description

Definition at line 1854 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::ExportQueryStmt::ExportQueryStmt ( std::string *  q,
std::string *  p,
std::list< NameValueAssign * > *  o 
)
inline

Definition at line 1856 of file ParserNode.h.

References options_.

1857  : select_stmt_(q), file_path_(p) {
1858  if (o) {
1859  for (const auto e : *o) {
1860  options_.emplace_back(e);
1861  }
1862  delete o;
1863  }
1864  }
std::unique_ptr< std::string > file_path_
Definition: ParserNode.h:1873
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1872
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1874
Parser::ExportQueryStmt::ExportQueryStmt ( const rapidjson::Value &  payload)

Definition at line 5599 of file ParserNode.cpp.

References CHECK, json_str(), and Parser::anonymous_namespace{ParserNode.cpp}::parse_options().

5599  {
5600  CHECK(payload.HasMember("filePath"));
5601  file_path_ = std::make_unique<std::string>(json_str(payload["filePath"]));
5602 
5603  CHECK(payload.HasMember("query"));
5604  select_stmt_ = std::make_unique<std::string>(json_str(payload["query"]));
5605 
5606  if ((*select_stmt_).back() != ';') {
5607  (*select_stmt_).push_back(';');
5608  }
5609  // Export wrapped everything with ` quotes which need cleanup
5610  boost::replace_all((*select_stmt_), "`", "");
5611 
5612  parse_options(payload, options_);
5613 }
const std::string json_str(const rapidjson::Value &obj) noexcept
Definition: JsonAccessors.h:44
std::unique_ptr< std::string > file_path_
Definition: ParserNode.h:1873
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1872
void parse_options(const rapidjson::Value &payload, std::list< std::unique_ptr< NameValueAssign >> &nameValueList, bool stringToNull=false, bool stringToInteger=false)
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1874
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

Member Function Documentation

void Parser::ExportQueryStmt::execute ( const Catalog_Namespace::SessionInfo session)
overridevirtual

Implements Parser::DDLStmt.

Definition at line 5615 of file ParserNode.cpp.

References import_export::QueryExporter::create(), query_state::QueryState::create(), ddl_utils::EXPORT, g_base_path, Catalog_Namespace::SessionInfo::get_session_id(), import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kNone, Parser::LocalConnector::query(), STDLOG, and ddl_utils::validate_allowed_file_path().

Referenced by omnisci.cursor.Cursor::executemany().

5615  {
5616  auto session_copy = session;
5617  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
5618  &session_copy, boost::null_deleter());
5619  auto query_state = query_state::QueryState::create(session_ptr, *select_stmt_);
5620  auto stdlog = STDLOG(query_state);
5621  auto query_state_proxy = query_state->createQueryStateProxy();
5622 
5623  LocalConnector local_connector;
5624 
5625  if (!leafs_connector_) {
5626  leafs_connector_ = &local_connector;
5627  }
5628 
5629  import_export::CopyParams copy_params;
5630  // @TODO(se) move rest to CopyParams when we have a Thrift endpoint
5633  std::string layer_name;
5638 
5639  parseOptions(copy_params, file_type, layer_name, file_compression, array_null_handling);
5640 
5641  if (file_path_->empty()) {
5642  throw std::runtime_error("Invalid file path for COPY TO");
5643  } else if (!boost::filesystem::path(*file_path_).is_absolute()) {
5644  std::string file_name = boost::filesystem::path(*file_path_).filename().string();
5645  std::string file_dir = g_base_path + "/mapd_export/" + session.get_session_id() + "/";
5646  if (!boost::filesystem::exists(file_dir)) {
5647  if (!boost::filesystem::create_directories(file_dir)) {
5648  throw std::runtime_error("Directory " + file_dir + " cannot be created.");
5649  }
5650  }
5651  *file_path_ = file_dir + file_name;
5652  } else {
5653  // Above branch will create a new file in the mapd_export directory. If that
5654  // path is not exercised, go through applicable file path validations.
5657  }
5658 
5659  // get column info
5660  auto column_info_result =
5661  local_connector.query(query_state_proxy, *select_stmt_, {}, true, false);
5662 
5663  // create exporter for requested file type
5664  auto query_exporter = import_export::QueryExporter::create(file_type);
5665 
5666  // default layer name to file path stem if it wasn't specified
5667  if (layer_name.size() == 0) {
5668  layer_name = boost::filesystem::path(*file_path_).stem().string();
5669  }
5670 
5671  // begin export
5672  query_exporter->beginExport(*file_path_,
5673  layer_name,
5674  copy_params,
5675  column_info_result.targets_meta,
5676  file_compression,
5677  array_null_handling);
5678 
5679  // how many fragments?
5680  size_t outer_frag_count =
5681  leafs_connector_->getOuterFragmentCount(query_state_proxy, *select_stmt_);
5682  size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
5683 
5684  // loop fragments
5685  for (size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
5686  // limit the query to just this fragment
5687  std::vector<size_t> allowed_outer_fragment_indices;
5688  if (outer_frag_count) {
5689  allowed_outer_fragment_indices.push_back(outer_frag_idx);
5690  }
5691 
5692  // run the query
5693  std::vector<AggregatedResult> query_results = leafs_connector_->query(
5694  query_state_proxy, *select_stmt_, allowed_outer_fragment_indices, false);
5695 
5696  // export the results
5697  query_exporter->exportResults(query_results);
5698  }
5699 
5700  // end export
5701  query_exporter->endExport();
5702 }
void parseOptions(import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
static std::shared_ptr< QueryState > create(ARGS &&...args)
Definition: QueryState.h:145
std::unique_ptr< std::string > file_path_
Definition: ParserNode.h:1873
DistributedConnector * leafs_connector_
Definition: ParserNode.h:1869
static std::unique_ptr< QueryExporter > create(const FileType file_type)
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1872
std::string g_base_path
Definition: SysCatalog.cpp:63
virtual size_t getOuterFragmentCount(QueryStateProxy, std::string &sql_query_string)=0
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:729
std::string get_session_id() const
Definition: SessionInfo.h:78
virtual std::vector< AggregatedResult > query(QueryStateProxy, std::string &sql_query_string, std::vector< size_t > outer_frag_indices, bool allow_interrupt)=0
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::string Parser::ExportQueryStmt::get_select_stmt ( ) const
inline

Definition at line 1867 of file ParserNode.h.

References select_stmt_.

Referenced by DBHandler::sql_execute_impl().

1867 { return *select_stmt_; }
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1872

+ Here is the caller graph for this function:

void Parser::ExportQueryStmt::parseOptions ( import_export::CopyParams copy_params,
import_export::QueryExporter::FileType file_type,
std::string &  layer_name,
import_export::QueryExporter::FileCompression file_compression,
import_export::QueryExporter::ArrayNullHandling array_null_handling 
)
private

Definition at line 5704 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::bool_from_string_literal(), import_export::CopyParams::delimiter, import_export::CopyParams::escape, Parser::StringLiteral::get_stringval(), import_export::HAS_HEADER, import_export::CopyParams::has_header, import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kExportSentinels, import_export::QueryExporter::kExportZeros, import_export::QueryExporter::kFlatGeobuf, import_export::QueryExporter::kGeoJSON, import_export::QueryExporter::kGeoJSONL, import_export::QueryExporter::kGZip, import_export::QueryExporter::kNone, import_export::QueryExporter::kNullEntireField, import_export::QueryExporter::kShapefile, import_export::QueryExporter::kZip, import_export::CopyParams::line_delim, import_export::NO_HEADER, import_export::CopyParams::null_str, import_export::CopyParams::quote, and import_export::CopyParams::quoted.

5709  {
5710  // defaults for non-CopyParams values
5712  layer_name.clear();
5714 
5715  if (!options_.empty()) {
5716  for (auto& p : options_) {
5717  if (boost::iequals(*p->get_name(), "delimiter")) {
5718  const StringLiteral* str_literal =
5719  dynamic_cast<const StringLiteral*>(p->get_value());
5720  if (str_literal == nullptr) {
5721  throw std::runtime_error("Delimiter option must be a string.");
5722  } else if (str_literal->get_stringval()->length() != 1) {
5723  throw std::runtime_error("Delimiter must be a single character string.");
5724  }
5725  copy_params.delimiter = (*str_literal->get_stringval())[0];
5726  } else if (boost::iequals(*p->get_name(), "nulls")) {
5727  const StringLiteral* str_literal =
5728  dynamic_cast<const StringLiteral*>(p->get_value());
5729  if (str_literal == nullptr) {
5730  throw std::runtime_error("Nulls option must be a string.");
5731  }
5732  copy_params.null_str = *str_literal->get_stringval();
5733  } else if (boost::iequals(*p->get_name(), "header")) {
5734  const StringLiteral* str_literal =
5735  dynamic_cast<const StringLiteral*>(p->get_value());
5736  if (str_literal == nullptr) {
5737  throw std::runtime_error("Header option must be a boolean.");
5738  }
5739  copy_params.has_header = bool_from_string_literal(str_literal)
5742  } else if (boost::iequals(*p->get_name(), "quote")) {
5743  const StringLiteral* str_literal =
5744  dynamic_cast<const StringLiteral*>(p->get_value());
5745  if (str_literal == nullptr) {
5746  throw std::runtime_error("Quote option must be a string.");
5747  } else if (str_literal->get_stringval()->length() != 1) {
5748  throw std::runtime_error("Quote must be a single character string.");
5749  }
5750  copy_params.quote = (*str_literal->get_stringval())[0];
5751  } else if (boost::iequals(*p->get_name(), "escape")) {
5752  const StringLiteral* str_literal =
5753  dynamic_cast<const StringLiteral*>(p->get_value());
5754  if (str_literal == nullptr) {
5755  throw std::runtime_error("Escape option must be a string.");
5756  } else if (str_literal->get_stringval()->length() != 1) {
5757  throw std::runtime_error("Escape must be a single character string.");
5758  }
5759  copy_params.escape = (*str_literal->get_stringval())[0];
5760  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
5761  const StringLiteral* str_literal =
5762  dynamic_cast<const StringLiteral*>(p->get_value());
5763  if (str_literal == nullptr) {
5764  throw std::runtime_error("Line_delimiter option must be a string.");
5765  } else if (str_literal->get_stringval()->length() != 1) {
5766  throw std::runtime_error("Line_delimiter must be a single character string.");
5767  }
5768  copy_params.line_delim = (*str_literal->get_stringval())[0];
5769  } else if (boost::iequals(*p->get_name(), "quoted")) {
5770  const StringLiteral* str_literal =
5771  dynamic_cast<const StringLiteral*>(p->get_value());
5772  if (str_literal == nullptr) {
5773  throw std::runtime_error("Quoted option must be a boolean.");
5774  }
5775  copy_params.quoted = bool_from_string_literal(str_literal);
5776  } else if (boost::iequals(*p->get_name(), "file_type")) {
5777  const StringLiteral* str_literal =
5778  dynamic_cast<const StringLiteral*>(p->get_value());
5779  if (str_literal == nullptr) {
5780  throw std::runtime_error("File Type option must be a string.");
5781  }
5782  auto file_type_str =
5783  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
5784  if (file_type_str == "csv") {
5786  } else if (file_type_str == "geojson") {
5788  } else if (file_type_str == "geojsonl") {
5790  } else if (file_type_str == "shapefile") {
5792  } else if (file_type_str == "flatgeobuf") {
5794  } else {
5795  throw std::runtime_error(
5796  "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL', "
5797  "'Shapefile', or 'FlatGeobuf'");
5798  }
5799  } else if (boost::iequals(*p->get_name(), "layer_name")) {
5800  const StringLiteral* str_literal =
5801  dynamic_cast<const StringLiteral*>(p->get_value());
5802  if (str_literal == nullptr) {
5803  throw std::runtime_error("Layer Name option must be a string.");
5804  }
5805  layer_name = *str_literal->get_stringval();
5806  } else if (boost::iequals(*p->get_name(), "file_compression")) {
5807  const StringLiteral* str_literal =
5808  dynamic_cast<const StringLiteral*>(p->get_value());
5809  if (str_literal == nullptr) {
5810  throw std::runtime_error("File Compression option must be a string.");
5811  }
5812  auto file_compression_str =
5813  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
5814  if (file_compression_str == "none") {
5816  } else if (file_compression_str == "gzip") {
5818  } else if (file_compression_str == "zip") {
5820  } else {
5821  throw std::runtime_error(
5822  "File Compression option must be 'None', 'GZip', or 'Zip'");
5823  }
5824  } else if (boost::iequals(*p->get_name(), "array_null_handling")) {
5825  const StringLiteral* str_literal =
5826  dynamic_cast<const StringLiteral*>(p->get_value());
5827  if (str_literal == nullptr) {
5828  throw std::runtime_error("Array Null Handling option must be a string.");
5829  }
5830  auto array_null_handling_str =
5831  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
5832  if (array_null_handling_str == "abort") {
5833  array_null_handling =
5835  } else if (array_null_handling_str == "raw") {
5836  array_null_handling =
5838  } else if (array_null_handling_str == "zero") {
5839  array_null_handling =
5841  } else if (array_null_handling_str == "nullfield") {
5842  array_null_handling =
5844  } else {
5845  throw std::runtime_error(
5846  "Array Null Handling option must be 'Abort', 'Raw', 'Zero', or "
5847  "'NullField'");
5848  }
5849  } else {
5850  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
5851  }
5852  }
5853  }
5854 }
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
Definition: ParserNode.cpp:945
ImportHeaderRow has_header
Definition: CopyParams.h:51
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1874

+ Here is the call graph for this function:

Member Data Documentation

std::unique_ptr<std::string> Parser::ExportQueryStmt::file_path_
private

Definition at line 1873 of file ParserNode.h.

DistributedConnector* Parser::ExportQueryStmt::leafs_connector_ = nullptr

Definition at line 1869 of file ParserNode.h.

std::list<std::unique_ptr<NameValueAssign> > Parser::ExportQueryStmt::options_
private

Definition at line 1874 of file ParserNode.h.

Referenced by ExportQueryStmt().

std::unique_ptr<std::string> Parser::ExportQueryStmt::select_stmt_
private

Definition at line 1872 of file ParserNode.h.

Referenced by get_select_stmt().


The documentation for this class was generated from the following files: