OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Parser::ExportQueryStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::ExportQueryStmt:
+ Collaboration diagram for Parser::ExportQueryStmt:

Public Member Functions

 ExportQueryStmt (std::string *q, std::string *p, std::list< NameValueAssign * > *o)
 
 ExportQueryStmt (const rapidjson::Value &payload)
 
void execute (const Catalog_Namespace::SessionInfo &session, bool read_only_mode) override
 
const std::string get_select_stmt () const
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

QueryConnectorleafs_connector_ = nullptr
 

Private Member Functions

void parseOptions (import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
 

Private Attributes

std::unique_ptr< std::string > select_stmt_
 
std::unique_ptr< std::string > file_path_
 
std::list< std::unique_ptr
< NameValueAssign > > 
options_
 

Detailed Description

Definition at line 1892 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::ExportQueryStmt::ExportQueryStmt ( std::string *  q,
std::string *  p,
std::list< NameValueAssign * > *  o 
)
inline

Definition at line 1894 of file ParserNode.h.

References options_.

1895  : select_stmt_(q), file_path_(p) {
1896  if (o) {
1897  for (const auto e : *o) {
1898  options_.emplace_back(e);
1899  }
1900  delete o;
1901  }
1902  }
std::unique_ptr< std::string > file_path_
Definition: ParserNode.h:1912
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1911
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1913
Parser::ExportQueryStmt::ExportQueryStmt ( const rapidjson::Value &  payload)

Definition at line 6060 of file ParserNode.cpp.

References CHECK, file_path_, json_str(), options_, Parser::anonymous_namespace{ParserNode.cpp}::parse_options(), and select_stmt_.

6060  {
6061  CHECK(payload.HasMember("filePath"));
6062  file_path_ = std::make_unique<std::string>(json_str(payload["filePath"]));
6063 
6064  CHECK(payload.HasMember("query"));
6065  select_stmt_ = std::make_unique<std::string>(json_str(payload["query"]));
6066 
6067  if ((*select_stmt_).back() != ';') {
6068  (*select_stmt_).push_back(';');
6069  }
6070  // Export wrapped everything with ` quotes which need cleanup
6071  boost::replace_all((*select_stmt_), "`", "");
6072 
6073  parse_options(payload, options_);
6074 }
const std::string json_str(const rapidjson::Value &obj) noexcept
Definition: JsonAccessors.h:44
std::unique_ptr< std::string > file_path_
Definition: ParserNode.h:1912
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1911
void parse_options(const rapidjson::Value &payload, std::list< std::unique_ptr< NameValueAssign >> &nameValueList, bool stringToNull=false, bool stringToInteger=false)
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1913
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

Member Function Documentation

void Parser::ExportQueryStmt::execute ( const Catalog_Namespace::SessionInfo session,
bool  read_only_mode 
)
overridevirtual

Implements Parser::DDLStmt.

Definition at line 6076 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::acquire_query_table_locks(), import_export::QueryExporter::create(), query_state::QueryState::create(), legacylockmgr::ExecutorOuterLock, ddl_utils::EXPORT, file_path_, g_base_path, Catalog_Namespace::SessionInfo::get_session_id(), legacylockmgr::LockMgr< MutexType, KeyType >::getMutex(), Parser::QueryConnector::getOuterFragmentCount(), import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, shared::kDefaultExportDirName, import_export::QueryExporter::kNone, leafs_connector_, parseOptions(), Parser::QueryConnector::query(), Parser::LocalQueryConnector::query(), select_stmt_, STDLOG, and ddl_utils::validate_allowed_file_path().

Referenced by heavydb.cursor.Cursor::executemany().

6077  {
6078  // valid in read_only_mode
6079  auto session_copy = session;
6080  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
6081  &session_copy, boost::null_deleter());
6082  auto query_state = query_state::QueryState::create(session_ptr, *select_stmt_);
6083  auto stdlog = STDLOG(query_state);
6084  auto query_state_proxy = query_state->createQueryStateProxy();
6085 
6086  LocalQueryConnector local_connector;
6087 
6088  if (!leafs_connector_) {
6089  leafs_connector_ = &local_connector;
6090  }
6091 
6092  import_export::CopyParams copy_params;
6093  // @TODO(se) move rest to CopyParams when we have a Thrift endpoint
6096  std::string layer_name;
6101 
6102  parseOptions(copy_params, file_type, layer_name, file_compression, array_null_handling);
6103 
6104  if (file_path_->empty()) {
6105  throw std::runtime_error("Invalid file path for COPY TO");
6106  } else if (!boost::filesystem::path(*file_path_).is_absolute()) {
6107  std::string file_name = boost::filesystem::path(*file_path_).filename().string();
6108  std::string file_dir = g_base_path + "/" + shared::kDefaultExportDirName + "/" +
6109  session.get_session_id() + "/";
6110  if (!boost::filesystem::exists(file_dir)) {
6111  if (!boost::filesystem::create_directories(file_dir)) {
6112  throw std::runtime_error("Directory " + file_dir + " cannot be created.");
6113  }
6114  }
6115  *file_path_ = file_dir + file_name;
6116  } else {
6117  // Above branch will create a new file in the export directory. If that
6118  // path is not exercised, go through applicable file path validations.
6121  }
6122 
6123  const auto execute_read_lock =
6127  auto locks = acquire_query_table_locks(
6128  session_ptr->getCatalog(), *select_stmt_, query_state_proxy);
6129 
6130  // get column info
6131  auto column_info_result =
6132  local_connector.query(query_state_proxy, *select_stmt_, {}, true, false);
6133 
6134  // create exporter for requested file type
6135  auto query_exporter = import_export::QueryExporter::create(file_type);
6136 
6137  // default layer name to file path stem if it wasn't specified
6138  if (layer_name.size() == 0) {
6139  layer_name = boost::filesystem::path(*file_path_).stem().string();
6140  }
6141 
6142  // begin export
6143  query_exporter->beginExport(*file_path_,
6144  layer_name,
6145  copy_params,
6146  column_info_result.targets_meta,
6147  file_compression,
6148  array_null_handling);
6149 
6150  // how many fragments?
6151  size_t outer_frag_count =
6152  leafs_connector_->getOuterFragmentCount(query_state_proxy, *select_stmt_);
6153  size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
6154 
6155  // loop fragments
6156  for (size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
6157  // limit the query to just this fragment
6158  std::vector<size_t> allowed_outer_fragment_indices;
6159  if (outer_frag_count) {
6160  allowed_outer_fragment_indices.push_back(outer_frag_idx);
6161  }
6162 
6163  // run the query
6164  std::vector<AggregatedResult> query_results = leafs_connector_->query(
6165  query_state_proxy, *select_stmt_, allowed_outer_fragment_indices, false);
6166 
6167  // export the results
6168  query_exporter->exportResults(query_results);
6169  }
6170 
6171  // end export
6172  query_exporter->endExport();
6173 }
static std::shared_ptr< WrapperType< MutexType > > getMutex(const LockType lockType, const KeyType &key)
void parseOptions(import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)
static std::shared_ptr< QueryState > create(ARGS &&...args)
Definition: QueryState.h:145
std::unique_ptr< std::string > file_path_
Definition: ParserNode.h:1912
const std::string kDefaultExportDirName
QueryConnector * leafs_connector_
Definition: ParserNode.h:1908
static std::unique_ptr< QueryExporter > create(const FileType file_type)
std::shared_lock< T > shared_lock
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1911
std::string g_base_path
Definition: SysCatalog.cpp:62
lockmgr::LockedTableDescriptors acquire_query_table_locks(Catalog_Namespace::Catalog &catalog, const std::string &query_str, const QueryStateProxy &query_state_proxy, const std::optional< std::string > &insert_table_name={})
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:770
std::string get_session_id() const
Definition: SessionInfo.h:76
virtual size_t getOuterFragmentCount(QueryStateProxy, std::string &sql_query_string)=0
virtual std::vector< AggregatedResult > query(QueryStateProxy, std::string &sql_query_string, std::vector< size_t > outer_frag_indices, bool allow_interrupt)=0
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const std::string Parser::ExportQueryStmt::get_select_stmt ( ) const
inline

Definition at line 1906 of file ParserNode.h.

References select_stmt_.

1906 { return *select_stmt_; }
std::unique_ptr< std::string > select_stmt_
Definition: ParserNode.h:1911
void Parser::ExportQueryStmt::parseOptions ( import_export::CopyParams copy_params,
import_export::QueryExporter::FileType file_type,
std::string &  layer_name,
import_export::QueryExporter::FileCompression file_compression,
import_export::QueryExporter::ArrayNullHandling array_null_handling 
)
private

Definition at line 6175 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::bool_from_string_literal(), import_export::CopyParams::delimiter, import_export::CopyParams::escape, Parser::StringLiteral::get_stringval(), import_export::CopyParams::has_header, import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kExportSentinels, import_export::QueryExporter::kExportZeros, import_export::QueryExporter::kFlatGeobuf, import_export::QueryExporter::kGeoJSON, import_export::QueryExporter::kGeoJSONL, import_export::QueryExporter::kGZip, import_export::kHasHeader, import_export::kNoHeader, import_export::QueryExporter::kNone, import_export::QueryExporter::kNullEntireField, import_export::QueryExporter::kShapefile, import_export::QueryExporter::kZip, import_export::CopyParams::line_delim, import_export::CopyParams::null_str, options_, import_export::CopyParams::quote, and import_export::CopyParams::quoted.

Referenced by execute().

6180  {
6181  // defaults for non-CopyParams values
6183  layer_name.clear();
6185 
6186  if (!options_.empty()) {
6187  for (auto& p : options_) {
6188  if (boost::iequals(*p->get_name(), "delimiter")) {
6189  const StringLiteral* str_literal =
6190  dynamic_cast<const StringLiteral*>(p->get_value());
6191  if (str_literal == nullptr) {
6192  throw std::runtime_error("Delimiter option must be a string.");
6193  } else if (str_literal->get_stringval()->length() != 1) {
6194  throw std::runtime_error("Delimiter must be a single character string.");
6195  }
6196  copy_params.delimiter = (*str_literal->get_stringval())[0];
6197  } else if (boost::iequals(*p->get_name(), "nulls")) {
6198  const StringLiteral* str_literal =
6199  dynamic_cast<const StringLiteral*>(p->get_value());
6200  if (str_literal == nullptr) {
6201  throw std::runtime_error("Nulls option must be a string.");
6202  }
6203  copy_params.null_str = *str_literal->get_stringval();
6204  } else if (boost::iequals(*p->get_name(), "header")) {
6205  const StringLiteral* str_literal =
6206  dynamic_cast<const StringLiteral*>(p->get_value());
6207  if (str_literal == nullptr) {
6208  throw std::runtime_error("Header option must be a boolean.");
6209  }
6210  copy_params.has_header = bool_from_string_literal(str_literal)
6213  } else if (boost::iequals(*p->get_name(), "quote")) {
6214  const StringLiteral* str_literal =
6215  dynamic_cast<const StringLiteral*>(p->get_value());
6216  if (str_literal == nullptr) {
6217  throw std::runtime_error("Quote option must be a string.");
6218  } else if (str_literal->get_stringval()->length() != 1) {
6219  throw std::runtime_error("Quote must be a single character string.");
6220  }
6221  copy_params.quote = (*str_literal->get_stringval())[0];
6222  } else if (boost::iequals(*p->get_name(), "escape")) {
6223  const StringLiteral* str_literal =
6224  dynamic_cast<const StringLiteral*>(p->get_value());
6225  if (str_literal == nullptr) {
6226  throw std::runtime_error("Escape option must be a string.");
6227  } else if (str_literal->get_stringval()->length() != 1) {
6228  throw std::runtime_error("Escape must be a single character string.");
6229  }
6230  copy_params.escape = (*str_literal->get_stringval())[0];
6231  } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
6232  const StringLiteral* str_literal =
6233  dynamic_cast<const StringLiteral*>(p->get_value());
6234  if (str_literal == nullptr) {
6235  throw std::runtime_error("Line_delimiter option must be a string.");
6236  } else if (str_literal->get_stringval()->length() != 1) {
6237  throw std::runtime_error("Line_delimiter must be a single character string.");
6238  }
6239  copy_params.line_delim = (*str_literal->get_stringval())[0];
6240  } else if (boost::iequals(*p->get_name(), "quoted")) {
6241  const StringLiteral* str_literal =
6242  dynamic_cast<const StringLiteral*>(p->get_value());
6243  if (str_literal == nullptr) {
6244  throw std::runtime_error("Quoted option must be a boolean.");
6245  }
6246  copy_params.quoted = bool_from_string_literal(str_literal);
6247  } else if (boost::iequals(*p->get_name(), "file_type")) {
6248  const StringLiteral* str_literal =
6249  dynamic_cast<const StringLiteral*>(p->get_value());
6250  if (str_literal == nullptr) {
6251  throw std::runtime_error("File Type option must be a string.");
6252  }
6253  auto file_type_str =
6254  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
6255  if (file_type_str == "csv") {
6257  } else if (file_type_str == "geojson") {
6259  } else if (file_type_str == "geojsonl") {
6261  } else if (file_type_str == "shapefile") {
6263  } else if (file_type_str == "flatgeobuf") {
6265  } else {
6266  throw std::runtime_error(
6267  "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL', "
6268  "'Shapefile', or 'FlatGeobuf'");
6269  }
6270  } else if (boost::iequals(*p->get_name(), "layer_name")) {
6271  const StringLiteral* str_literal =
6272  dynamic_cast<const StringLiteral*>(p->get_value());
6273  if (str_literal == nullptr) {
6274  throw std::runtime_error("Layer Name option must be a string.");
6275  }
6276  layer_name = *str_literal->get_stringval();
6277  } else if (boost::iequals(*p->get_name(), "file_compression")) {
6278  const StringLiteral* str_literal =
6279  dynamic_cast<const StringLiteral*>(p->get_value());
6280  if (str_literal == nullptr) {
6281  throw std::runtime_error("File Compression option must be a string.");
6282  }
6283  auto file_compression_str =
6284  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
6285  if (file_compression_str == "none") {
6287  } else if (file_compression_str == "gzip") {
6289  } else if (file_compression_str == "zip") {
6291  } else {
6292  throw std::runtime_error(
6293  "File Compression option must be 'None', 'GZip', or 'Zip'");
6294  }
6295  } else if (boost::iequals(*p->get_name(), "array_null_handling")) {
6296  const StringLiteral* str_literal =
6297  dynamic_cast<const StringLiteral*>(p->get_value());
6298  if (str_literal == nullptr) {
6299  throw std::runtime_error("Array Null Handling option must be a string.");
6300  }
6301  auto array_null_handling_str =
6302  boost::algorithm::to_lower_copy(*str_literal->get_stringval());
6303  if (array_null_handling_str == "abort") {
6304  array_null_handling =
6306  } else if (array_null_handling_str == "raw") {
6307  array_null_handling =
6309  } else if (array_null_handling_str == "zero") {
6310  array_null_handling =
6312  } else if (array_null_handling_str == "nullfield") {
6313  array_null_handling =
6315  } else {
6316  throw std::runtime_error(
6317  "Array Null Handling option must be 'Abort', 'Raw', 'Zero', or "
6318  "'NullField'");
6319  }
6320  } else {
6321  throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
6322  }
6323  }
6324  }
6325 }
bool bool_from_string_literal(const Parser::StringLiteral *str_literal)
ImportHeaderRow has_header
Definition: CopyParams.h:46
std::list< std::unique_ptr< NameValueAssign > > options_
Definition: ParserNode.h:1913

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::unique_ptr<std::string> Parser::ExportQueryStmt::file_path_
private

Definition at line 1912 of file ParserNode.h.

Referenced by execute(), and ExportQueryStmt().

QueryConnector* Parser::ExportQueryStmt::leafs_connector_ = nullptr

Definition at line 1908 of file ParserNode.h.

Referenced by execute().

std::list<std::unique_ptr<NameValueAssign> > Parser::ExportQueryStmt::options_
private

Definition at line 1913 of file ParserNode.h.

Referenced by ExportQueryStmt(), and parseOptions().

std::unique_ptr<std::string> Parser::ExportQueryStmt::select_stmt_
private

Definition at line 1911 of file ParserNode.h.

Referenced by execute(), ExportQueryStmt(), and get_select_stmt().


The documentation for this class was generated from the following files: