Parser::ExportQueryStmt Class Reference

#include <ParserNode.h>

Inheritance diagram for Parser::ExportQueryStmt:

Collaboration diagram for Parser::ExportQueryStmt:

Public Member Functions
	ExportQueryStmt (std::string *q, std::string *p, std::list< NameValueAssign * > *o)
	ExportQueryStmt (const rapidjson::Value &payload)
void	execute (const Catalog_Namespace::SessionInfo &session, bool read_only_mode) override
const std::string	get_select_stmt () const
Public Member Functions inherited from Parser::DDLStmt
void	setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
Public Member Functions inherited from Parser::Node
virtual	~Node ()

Public Attributes
std::unique_ptr< QueryConnector >	leafs_connector_

Private Member Functions
void	parseOptions (import_export::CopyParams &copy_params, import_export::QueryExporter::FileType &file_type, std::string &layer_name, import_export::QueryExporter::FileCompression &file_compression, import_export::QueryExporter::ArrayNullHandling &array_null_handling)

Private Attributes
std::unique_ptr< std::string >	select_stmt_
std::unique_ptr< std::string >	file_path_
std::list< std::unique_ptr < NameValueAssign > >	options_

Detailed Description

Definition at line 1848 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::ExportQueryStmt::ExportQueryStmt ( std::string *  q, std::string *  p, std::list< NameValueAssign * > *  o  )

inline

Definition at line 1850 of file ParserNode.h.

References options_.

      : select_stmt_(q), file_path_(p) {
    if (o) {
      for (const auto e : *o) {
        options_.emplace_back(e);
      }
      delete o;
    }
  }

Parser::ExportQueryStmt::ExportQueryStmt

(

const rapidjson::Value &

payload

)

Definition at line 6138 of file ParserNode.cpp.

References CHECK, file_path_, json_str(), options_, Parser::anonymous_namespace{ParserNode.cpp}::parse_options(), and select_stmt_.

                                                              {
  CHECK(payload.HasMember("filePath"));
  file_path_ = std::make_unique<std::string>(json_str(payload["filePath"]));

  CHECK(payload.HasMember("query"));
  select_stmt_ = std::make_unique<std::string>(json_str(payload["query"]));

  if ((*select_stmt_).back() != ';') {
    (*select_stmt_).push_back(';');
  }
  // Export wrapped everything with ` quotes which need cleanup
  boost::replace_all((*select_stmt_), "`", "");

  parse_options(payload, options_);
}

Here is the call graph for this function:

Member Function Documentation

void Parser::ExportQueryStmt::execute ( const Catalog_Namespace::SessionInfo &  session, bool  read_only_mode  )

overridevirtual

Implements Parser::DDLStmt.

Definition at line 6154 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::acquire_query_table_locks(), import_export::QueryExporter::create(), query_state::QueryState::create(), query_state::QueryState::createQueryStateProxy(), legacylockmgr::ExecutorOuterLock, ddl_utils::EXPORT, file_path_, g_base_path, Catalog_Namespace::SessionInfo::get_session_id(), legacylockmgr::LockMgr< MutexType, KeyType >::getMutex(), import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, shared::kDefaultExportDirName, import_export::QueryExporter::kNone, leafs_connector_, parseOptions(), Parser::LocalQueryConnector::query(), select_stmt_, STDLOG, and ddl_utils::validate_allowed_file_path().

Referenced by heavydb.cursor.Cursor::executemany().

                                                   {
  // valid in read_only_mode
  auto session_copy = session;
  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
      &session_copy, boost::null_deleter());
  auto query_state = query_state::QueryState::create(session_ptr, *select_stmt_);
  auto stdlog = STDLOG(query_state);
  auto query_state_proxy = query_state->createQueryStateProxy();

  if (!leafs_connector_) {
    leafs_connector_ = std::make_unique<LocalQueryConnector>();
  }

  import_export::CopyParams copy_params;
  // @TODO(se) move rest to CopyParams when we have a Thrift endpoint
  import_export::QueryExporter::FileType file_type =
      import_export::QueryExporter::FileType::kCSV;
  std::string layer_name;
  import_export::QueryExporter::FileCompression file_compression =
      import_export::QueryExporter::FileCompression::kNone;
  import_export::QueryExporter::ArrayNullHandling array_null_handling =
      import_export::QueryExporter::ArrayNullHandling::kAbortWithWarning;

  parseOptions(copy_params, file_type, layer_name, file_compression, array_null_handling);

  if (file_path_->empty()) {
    throw std::runtime_error("Invalid file path for COPY TO");
  } else if (!boost::filesystem::path(*file_path_).is_absolute()) {
    std::string file_name = boost::filesystem::path(*file_path_).filename().string();
    std::string file_dir = g_base_path + "/" + shared::kDefaultExportDirName + "/" +
                           session.get_session_id() + "/";
    if (!boost::filesystem::exists(file_dir)) {
      if (!boost::filesystem::create_directories(file_dir)) {
        throw std::runtime_error("Directory " + file_dir + " cannot be created.");
      }
    }
    *file_path_ = file_dir + file_name;
  } else {
    // Above branch will create a new file in the export directory. If that
    // path is not exercised, go through applicable file path validations.
    ddl_utils::validate_allowed_file_path(*file_path_,
                                          ddl_utils::DataTransferType::EXPORT);
  }

  const auto execute_read_lock =
      heavyai::shared_lock<legacylockmgr::WrapperType<heavyai::shared_mutex>>(
          *legacylockmgr::LockMgr<heavyai::shared_mutex, bool>::getMutex(
              legacylockmgr::ExecutorOuterLock, true));
  auto locks = acquire_query_table_locks(
      session_ptr->getCatalog().name(), *select_stmt_, query_state_proxy);

  // get column info
  LocalQueryConnector local_connector;
  auto column_info_result =
      local_connector.query(query_state_proxy, *select_stmt_, {}, true, false);

  // create exporter for requested file type
  auto query_exporter = import_export::QueryExporter::create(file_type);

  // default layer name to file path stem if it wasn't specified
  if (layer_name.size() == 0) {
    layer_name = boost::filesystem::path(*file_path_).stem().string();
  }

  // begin export
  query_exporter->beginExport(*file_path_,
                              layer_name,
                              copy_params,
                              column_info_result.targets_meta,
                              file_compression,
                              array_null_handling);

  // how many fragments?
  size_t outer_frag_count =
      leafs_connector_->getOuterFragmentCount(query_state_proxy, *select_stmt_);
  size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;

  // loop fragments
  for (size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
    // limit the query to just this fragment
    std::vector<size_t> allowed_outer_fragment_indices;
    if (outer_frag_count) {
      allowed_outer_fragment_indices.push_back(outer_frag_idx);
    }

    // run the query
    std::vector<AggregatedResult> query_results = leafs_connector_->query(
        query_state_proxy, *select_stmt_, allowed_outer_fragment_indices, false);

    // export the results
    query_exporter->exportResults(query_results);
  }

  // end export
  query_exporter->endExport();
}

Here is the call graph for this function:

Here is the caller graph for this function:

const std::string Parser::ExportQueryStmt::get_select_stmt ( ) const

inline

Definition at line 1862 of file ParserNode.h.

References select_stmt_.

{ return *select_stmt_; }

void Parser::ExportQueryStmt::parseOptions ( import_export::CopyParams &  copy_params, import_export::QueryExporter::FileType &  file_type, std::string &  layer_name, import_export::QueryExporter::FileCompression &  file_compression, import_export::QueryExporter::ArrayNullHandling &  array_null_handling  )

private

Definition at line 6252 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::bool_from_string_literal(), import_export::CopyParams::delimiter, import_export::CopyParams::escape, Parser::StringLiteral::get_stringval(), import_export::CopyParams::has_header, import_export::QueryExporter::kAbortWithWarning, import_export::QueryExporter::kCSV, import_export::QueryExporter::kExportSentinels, import_export::QueryExporter::kExportZeros, import_export::QueryExporter::kFlatGeobuf, import_export::QueryExporter::kGeoJSON, import_export::QueryExporter::kGeoJSONL, import_export::QueryExporter::kGZip, import_export::kHasHeader, import_export::kNoHeader, import_export::QueryExporter::kNone, import_export::QueryExporter::kNullEntireField, import_export::QueryExporter::kShapefile, import_export::QueryExporter::kZip, import_export::CopyParams::line_delim, import_export::CopyParams::null_str, options_, import_export::CopyParams::quote, and import_export::CopyParams::quoted.

Referenced by execute().

                                                                      {
  // defaults for non-CopyParams values
  file_type = import_export::QueryExporter::FileType::kCSV;
  layer_name.clear();
  file_compression = import_export::QueryExporter::FileCompression::kNone;

  if (!options_.empty()) {
    for (auto& p : options_) {
      if (boost::iequals(*p->get_name(), "delimiter")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Delimiter option must be a string.");
        } else if (str_literal->get_stringval()->length() != 1) {
          throw std::runtime_error("Delimiter must be a single character string.");
        }
        copy_params.delimiter = (*str_literal->get_stringval())[0];
      } else if (boost::iequals(*p->get_name(), "nulls")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Nulls option must be a string.");
        }
        copy_params.null_str = *str_literal->get_stringval();
      } else if (boost::iequals(*p->get_name(), "header")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Header option must be a boolean.");
        }
        copy_params.has_header = bool_from_string_literal(str_literal)
                                     ? import_export::ImportHeaderRow::kHasHeader
                                     : import_export::ImportHeaderRow::kNoHeader;
      } else if (boost::iequals(*p->get_name(), "quote")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Quote option must be a string.");
        } else if (str_literal->get_stringval()->length() != 1) {
          throw std::runtime_error("Quote must be a single character string.");
        }
        copy_params.quote = (*str_literal->get_stringval())[0];
      } else if (boost::iequals(*p->get_name(), "escape")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Escape option must be a string.");
        } else if (str_literal->get_stringval()->length() != 1) {
          throw std::runtime_error("Escape must be a single character string.");
        }
        copy_params.escape = (*str_literal->get_stringval())[0];
      } else if (boost::iequals(*p->get_name(), "line_delimiter")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Line_delimiter option must be a string.");
        } else if (str_literal->get_stringval()->length() != 1) {
          throw std::runtime_error("Line_delimiter must be a single character string.");
        }
        copy_params.line_delim = (*str_literal->get_stringval())[0];
      } else if (boost::iequals(*p->get_name(), "quoted")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Quoted option must be a boolean.");
        }
        copy_params.quoted = bool_from_string_literal(str_literal);
      } else if (boost::iequals(*p->get_name(), "file_type")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("File Type option must be a string.");
        }
        auto file_type_str =
            boost::algorithm::to_lower_copy(*str_literal->get_stringval());
        if (file_type_str == "csv") {
          file_type = import_export::QueryExporter::FileType::kCSV;
        } else if (file_type_str == "geojson") {
          file_type = import_export::QueryExporter::FileType::kGeoJSON;
        } else if (file_type_str == "geojsonl") {
          file_type = import_export::QueryExporter::FileType::kGeoJSONL;
        } else if (file_type_str == "shapefile") {
          file_type = import_export::QueryExporter::FileType::kShapefile;
        } else if (file_type_str == "flatgeobuf") {
          file_type = import_export::QueryExporter::FileType::kFlatGeobuf;
        } else {
          throw std::runtime_error(
              "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL', "
              "'Shapefile', or 'FlatGeobuf'");
        }
      } else if (boost::iequals(*p->get_name(), "layer_name")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Layer Name option must be a string.");
        }
        layer_name = *str_literal->get_stringval();
      } else if (boost::iequals(*p->get_name(), "file_compression")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("File Compression option must be a string.");
        }
        auto file_compression_str =
            boost::algorithm::to_lower_copy(*str_literal->get_stringval());
        if (file_compression_str == "none") {
          file_compression = import_export::QueryExporter::FileCompression::kNone;
        } else if (file_compression_str == "gzip") {
          file_compression = import_export::QueryExporter::FileCompression::kGZip;
        } else if (file_compression_str == "zip") {
          file_compression = import_export::QueryExporter::FileCompression::kZip;
        } else {
          throw std::runtime_error(
              "File Compression option must be 'None', 'GZip', or 'Zip'");
        }
      } else if (boost::iequals(*p->get_name(), "array_null_handling")) {
        const StringLiteral* str_literal =
            dynamic_cast<const StringLiteral*>(p->get_value());
        if (str_literal == nullptr) {
          throw std::runtime_error("Array Null Handling option must be a string.");
        }
        auto array_null_handling_str =
            boost::algorithm::to_lower_copy(*str_literal->get_stringval());
        if (array_null_handling_str == "abort") {
          array_null_handling =
              import_export::QueryExporter::ArrayNullHandling::kAbortWithWarning;
        } else if (array_null_handling_str == "raw") {
          array_null_handling =
              import_export::QueryExporter::ArrayNullHandling::kExportSentinels;
        } else if (array_null_handling_str == "zero") {
          array_null_handling =
              import_export::QueryExporter::ArrayNullHandling::kExportZeros;
        } else if (array_null_handling_str == "nullfield") {
          array_null_handling =
              import_export::QueryExporter::ArrayNullHandling::kNullEntireField;
        } else {
          throw std::runtime_error(
              "Array Null Handling option must be 'Abort', 'Raw', 'Zero', or "
              "'NullField'");
        }
      } else {
        throw std::runtime_error("Invalid option for COPY: " + *p->get_name());
      }
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

std::unique_ptr<std::string> Parser::ExportQueryStmt::file_path_

private

Definition at line 1868 of file ParserNode.h.

Referenced by execute(), and ExportQueryStmt().

std::unique_ptr<QueryConnector> Parser::ExportQueryStmt::leafs_connector_

Definition at line 1864 of file ParserNode.h.

Referenced by execute().

std::list<std::unique_ptr<NameValueAssign> > Parser::ExportQueryStmt::options_

private

Definition at line 1869 of file ParserNode.h.

Referenced by ExportQueryStmt(), and parseOptions().

std::unique_ptr<std::string> Parser::ExportQueryStmt::select_stmt_

private

Definition at line 1867 of file ParserNode.h.

Referenced by execute(), ExportQueryStmt(), and get_select_stmt().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/Parser/ParserNode.h
• /home/jenkins-slave/workspace/core-os-doxygen/Parser/ParserNode.cpp