OmniSciDB  c0231cc57d
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Parser::InsertIntoTableAsSelectStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::InsertIntoTableAsSelectStmt:
+ Collaboration diagram for Parser::InsertIntoTableAsSelectStmt:

Public Member Functions

 InsertIntoTableAsSelectStmt (const rapidjson::Value &payload)
 
 InsertIntoTableAsSelectStmt (const std::string *table_name, const std::string *select_query, std::list< std::string * > *c)
 
void populateData (QueryStateProxy, const TableDescriptor *td, bool validate_table, bool for_CTAS=false)
 
void execute (const Catalog_Namespace::SessionInfo &session, bool read_only_mode) override
 
std::string & get_table ()
 
std::string & get_select_query ()
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

std::unique_ptr< QueryConnectorleafs_connector_
 

Protected Attributes

std::vector< std::unique_ptr
< std::string > > 
column_list_
 
std::string table_name_
 
std::string select_query_
 

Detailed Description

Definition at line 1126 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::InsertIntoTableAsSelectStmt::InsertIntoTableAsSelectStmt ( const rapidjson::Value &  payload)

Definition at line 3594 of file ParserNode.cpp.

References CHECK, column_list_, json_str(), select_query_, and table_name_.

3595  {
3596  CHECK(payload.HasMember("name"));
3597  table_name_ = json_str(payload["name"]);
3598 
3599  CHECK(payload.HasMember("query"));
3600  select_query_ = json_str(payload["query"]);
3601 
3602  boost::replace_all(select_query_, "\n", " ");
3603  select_query_ = "(" + select_query_ + ")";
3604 
3605  if (payload.HasMember("columns")) {
3606  CHECK(payload["columns"].IsArray());
3607  for (auto& column : payload["columns"].GetArray()) {
3608  std::string s = json_str(column);
3609  column_list_.emplace_back(std::unique_ptr<std::string>(new std::string(s)));
3610  }
3611  }
3612 }
const std::string json_str(const rapidjson::Value &obj) noexcept
Definition: JsonAccessors.h:44
std::vector< std::unique_ptr< std::string > > column_list_
Definition: ParserNode.h:1159
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

Parser::InsertIntoTableAsSelectStmt::InsertIntoTableAsSelectStmt ( const std::string *  table_name,
const std::string *  select_query,
std::list< std::string * > *  c 
)
inline

Definition at line 1130 of file ParserNode.h.

References column_list_.

1133  : table_name_(*table_name), select_query_(*select_query) {
1134  if (c) {
1135  for (auto e : *c) {
1136  column_list_.emplace_back(e);
1137  }
1138  delete c;
1139  }
1140 
1141  delete table_name;
1142  delete select_query;
1143  }
std::vector< std::unique_ptr< std::string > > column_list_
Definition: ParserNode.h:1159

Member Function Documentation

void Parser::InsertIntoTableAsSelectStmt::execute ( const Catalog_Namespace::SessionInfo session,
bool  read_only_mode 
)
overridevirtual

Implements Parser::DDLStmt.

Reimplemented in Parser::CreateTableAsSelectStmt.

Definition at line 4180 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::acquire_query_table_locks(), Executor::clearExternalCaches(), query_state::QueryState::create(), Catalog_Namespace::DBMetadata::dbId, legacylockmgr::ExecutorOuterLock, Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getMetadataForTable(), legacylockmgr::LockMgr< MutexType, KeyType >::getMutex(), populateData(), select_query_, STDLOG, and table_name_.

Referenced by heavydb.cursor.Cursor::executemany().

4181  {
4182  if (read_only_mode) {
4183  throw std::runtime_error("INSERT INTO TABLE invalid in read only mode.");
4184  }
4185  auto session_copy = session;
4186  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4187  &session_copy, boost::null_deleter());
4188  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
4189  auto stdlog = STDLOG(query_state);
4190  auto& catalog = session_ptr->getCatalog();
4191 
4192  const auto execute_read_lock =
4196 
4197  if (catalog.getMetadataForTable(table_name_) == nullptr) {
4198  throw std::runtime_error("ITAS failed: table " + table_name_ + " does not exist.");
4199  }
4200 
4201  auto locks = acquire_query_table_locks(
4202  catalog, select_query_, query_state->createQueryStateProxy(), table_name_);
4203  const TableDescriptor* td = catalog.getMetadataForTable(table_name_);
4204 
4205  Executor::clearExternalCaches(true, td, catalog.getCurrentDB().dbId);
4206 
4207  try {
4208  populateData(query_state->createQueryStateProxy(), td, true, false);
4209  } catch (...) {
4210  throw;
4211  }
4212 }
static std::shared_ptr< WrapperType< MutexType > > getMutex(const LockType lockType, const KeyType &key)
void populateData(QueryStateProxy, const TableDescriptor *td, bool validate_table, bool for_CTAS=false)
static std::shared_ptr< QueryState > create(ARGS &&...args)
Definition: QueryState.h:145
std::shared_lock< T > shared_lock
lockmgr::LockedTableDescriptors acquire_query_table_locks(Catalog_Namespace::Catalog &catalog, const std::string &query_str, const QueryStateProxy &query_state_proxy, const std::optional< std::string > &insert_table_name={})
static void clearExternalCaches(bool for_update, const TableDescriptor *td, const int current_db_id)
Definition: Execute.h:391
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string& Parser::InsertIntoTableAsSelectStmt::get_select_query ( )
inline

Definition at line 1154 of file ParserNode.h.

References select_query_.

1154 { return select_query_; }
std::string& Parser::InsertIntoTableAsSelectStmt::get_table ( )
inline

Definition at line 1152 of file ParserNode.h.

References table_name_.

1152 { return table_name_; }
void Parser::InsertIntoTableAsSelectStmt::populateData ( QueryStateProxy  query_state_proxy,
const TableDescriptor td,
bool  validate_table,
bool  for_CTAS = false 
)

Definition at line 3614 of file ParserNode.cpp.

References threading_serial::async(), CHECK, Parser::check_session_interrupted(), column_list_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, TargetValueConverterFactory::create(), Fragmenter_Namespace::InsertData::databaseId, logger::ERROR, import_export::fill_missing_columns(), g_cluster, g_enable_assign_render_groups, g_enable_non_kernel_time_query_interrupt, g_enable_string_functions, ResultSet::GeoTargetValue, SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), Parser::LocalQueryConnector::getColumnDescriptors(), query_state::QueryState::getConstSessionInfo(), Executor::getExecutor(), query_state::QueryStateProxy::getQueryState(), query_state::QueryState::getQuerySubmittedTime(), AccessPrivileges::INSERT_INTO_TABLE, Fragmenter_Namespace::InsertDataLoader::insertData(), SQLTypeInfo::is_array(), SQLTypeInfo::is_date(), SQLTypeInfo::is_decimal(), IS_GEO_POLY, SQLTypeInfo::is_geometry(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), SQLTypeInfo::is_timestamp(), TableDescriptor::isView, leafs_connector_, LOG, Fragmenter_Namespace::InsertData::numRows, Parser::LocalQueryConnector::query(), run_benchmark_import::res, run_benchmark_import::result, select_query_, run_benchmark_import::start_time, table_is_temporary(), table_name_, TableDBObjectType, TableDescriptor::tableId, Fragmenter_Namespace::InsertData::tableId, logger::thread_id(), timer_start(), timer_stop(), Executor::UNITARY_EXECUTOR_ID, UNLIKELY, foreign_storage::validate_non_foreign_table_write(), and VLOG.

Referenced by execute(), and Parser::CreateTableAsSelectStmt::execute().

3617  {
3618  auto const session = query_state_proxy.getQueryState().getConstSessionInfo();
3619  auto& catalog = session->getCatalog();
3621  bool populate_table = false;
3622 
3623  if (leafs_connector_) {
3624  populate_table = true;
3625  } else {
3626  leafs_connector_ = std::make_unique<LocalQueryConnector>();
3627  if (!g_cluster) {
3628  populate_table = true;
3629  }
3630  }
3631 
3632  auto get_target_column_descriptors = [this, &catalog](const TableDescriptor* td) {
3633  std::vector<const ColumnDescriptor*> target_column_descriptors;
3634  if (column_list_.empty()) {
3635  auto list = catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
3636  target_column_descriptors = {std::begin(list), std::end(list)};
3637  } else {
3638  for (auto& c : column_list_) {
3639  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
3640  if (cd == nullptr) {
3641  throw std::runtime_error("Column " + *c + " does not exist.");
3642  }
3643  target_column_descriptors.push_back(cd);
3644  }
3645  }
3646 
3647  return target_column_descriptors;
3648  };
3649 
3650  bool is_temporary = table_is_temporary(td);
3651 
3652  if (validate_table) {
3653  // check access privileges
3654  if (!td) {
3655  throw std::runtime_error("Table " + table_name_ + " does not exist.");
3656  }
3657  if (td->isView) {
3658  throw std::runtime_error("Insert to views is not supported yet.");
3659  }
3660 
3661  if (!session->checkDBAccessPrivileges(DBObjectType::TableDBObjectType,
3663  table_name_)) {
3664  throw std::runtime_error("User has no insert privileges on " + table_name_ + ".");
3665  }
3666 
3667  // only validate the select query so we get the target types
3668  // correctly, but do not populate the result set
3669  LocalQueryConnector local_connector;
3670  auto result = local_connector.query(query_state_proxy, select_query_, {}, true, true);
3671  auto source_column_descriptors = local_connector.getColumnDescriptors(result, false);
3672 
3673  std::vector<const ColumnDescriptor*> target_column_descriptors =
3674  get_target_column_descriptors(td);
3675 
3676  if (source_column_descriptors.size() != target_column_descriptors.size()) {
3677  throw std::runtime_error("The number of source and target columns does not match.");
3678  }
3679 
3680  for (int i = 0; i < source_column_descriptors.size(); i++) {
3681  const ColumnDescriptor* source_cd =
3682  &(*std::next(source_column_descriptors.begin(), i));
3683  const ColumnDescriptor* target_cd = target_column_descriptors.at(i);
3684 
3685  if (source_cd->columnType.get_type() != target_cd->columnType.get_type()) {
3686  auto type_cannot_be_cast = [](const auto& col_type) {
3687  return (col_type.is_time() || col_type.is_geometry() || col_type.is_array() ||
3688  col_type.is_boolean());
3689  };
3690 
3691  if (type_cannot_be_cast(source_cd->columnType) ||
3692  type_cannot_be_cast(target_cd->columnType)) {
3693  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3694  source_cd->columnType.get_type_name() +
3695  "' and target '" + target_cd->columnName + " " +
3696  target_cd->columnType.get_type_name() +
3697  "' column types do not match.");
3698  }
3699  }
3700  if (source_cd->columnType.is_array()) {
3701  if (source_cd->columnType.get_subtype() != target_cd->columnType.get_subtype()) {
3702  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3703  source_cd->columnType.get_type_name() +
3704  "' and target '" + target_cd->columnName + " " +
3705  target_cd->columnType.get_type_name() +
3706  "' array column element types do not match.");
3707  }
3708  }
3709 
3710  if (target_cd->columnType.is_string() && !source_cd->columnType.is_string()) {
3711  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3712  source_cd->columnType.get_type_name() +
3713  "' and target '" + target_cd->columnName + " " +
3714  target_cd->columnType.get_type_name() +
3715  "' column types do not match.");
3716  }
3717 
3718  if (source_cd->columnType.is_decimal() ||
3719  source_cd->columnType.get_elem_type().is_decimal()) {
3720  SQLTypeInfo sourceType = source_cd->columnType;
3721  SQLTypeInfo targetType = target_cd->columnType;
3722 
3723  if (source_cd->columnType.is_array()) {
3724  sourceType = source_cd->columnType.get_elem_type();
3725  targetType = target_cd->columnType.get_elem_type();
3726  }
3727 
3728  if (sourceType.get_scale() != targetType.get_scale()) {
3729  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3730  source_cd->columnType.get_type_name() +
3731  "' and target '" + target_cd->columnName + " " +
3732  target_cd->columnType.get_type_name() +
3733  "' decimal columns scales do not match.");
3734  }
3735  }
3736 
3737  if (source_cd->columnType.is_string()) {
3738  if (!target_cd->columnType.is_string()) {
3739  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3740  source_cd->columnType.get_type_name() +
3741  "' and target '" + target_cd->columnName + " " +
3742  target_cd->columnType.get_type_name() +
3743  "' column types do not match.");
3744  }
3745  if (source_cd->columnType.get_compression() !=
3746  target_cd->columnType.get_compression()) {
3747  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3748  source_cd->columnType.get_type_name() +
3749  "' and target '" + target_cd->columnName + " " +
3750  target_cd->columnType.get_type_name() +
3751  "' columns string encodings do not match.");
3752  }
3753  }
3754 
3755  if (source_cd->columnType.is_timestamp() && target_cd->columnType.is_timestamp()) {
3756  if (source_cd->columnType.get_dimension() !=
3757  target_cd->columnType.get_dimension()) {
3758  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3759  source_cd->columnType.get_type_name() +
3760  "' and target '" + target_cd->columnName + " " +
3761  target_cd->columnType.get_type_name() +
3762  "' timestamp column precisions do not match.");
3763  }
3764  }
3765 
3766  if (!source_cd->columnType.is_string() && !source_cd->columnType.is_geometry() &&
3767  !source_cd->columnType.is_integer() && !source_cd->columnType.is_decimal() &&
3768  !source_cd->columnType.is_date() && !source_cd->columnType.is_time() &&
3769  !source_cd->columnType.is_timestamp() &&
3770  source_cd->columnType.get_size() > target_cd->columnType.get_size()) {
3771  throw std::runtime_error("Source '" + source_cd->columnName + " " +
3772  source_cd->columnType.get_type_name() +
3773  "' and target '" + target_cd->columnName + " " +
3774  target_cd->columnType.get_type_name() +
3775  "' column encoding sizes do not match.");
3776  }
3777  }
3778  }
3779 
3780  if (!populate_table) {
3781  return;
3782  }
3783 
3784  int64_t total_row_count = 0;
3785  int64_t total_source_query_time_ms = 0;
3786  int64_t total_target_value_translate_time_ms = 0;
3787  int64_t total_data_load_time_ms = 0;
3788 
3790  auto target_column_descriptors = get_target_column_descriptors(td);
3791  auto outer_frag_count =
3792  leafs_connector_->getOuterFragmentCount(query_state_proxy, select_query_);
3793 
3794  size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
3795  auto query_session = session ? session->get_session_id() : "";
3797  std::string work_type_str = for_CTAS ? "CTAS" : "ITAS";
3798  try {
3799  for (size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
3800  std::vector<size_t> allowed_outer_fragment_indices;
3801 
3802  if (outer_frag_count) {
3803  allowed_outer_fragment_indices.push_back(outer_frag_idx);
3804  }
3805 
3806  const auto query_clock_begin = timer_start();
3807  std::vector<AggregatedResult> query_results =
3808  leafs_connector_->query(query_state_proxy,
3809  select_query_,
3810  allowed_outer_fragment_indices,
3812  total_source_query_time_ms += timer_stop(query_clock_begin);
3813 
3814  auto start_time = query_state_proxy.getQueryState().getQuerySubmittedTime();
3815  auto query_str = "INSERT_DATA for " + work_type_str;
3817  // In the clean-up phase of the query execution for collecting aggregated result
3818  // of SELECT query, we remove its query session info, so we need to enroll the
3819  // session info again
3820  executor->enrollQuerySession(query_session,
3821  query_str,
3822  start_time,
3824  QuerySessionStatus::QueryStatus::RUNNING_IMPORTER);
3825  }
3826 
3827  ScopeGuard clearInterruptStatus = [executor, &query_session, &start_time] {
3828  // this data population is non-kernel operation, so we manually cleanup
3829  // the query session info in the cleanup phase
3831  executor->clearQuerySessionStatus(query_session, start_time);
3832  }
3833  };
3834 
3835  for (auto& res : query_results) {
3836  if (UNLIKELY(check_session_interrupted(query_session, executor))) {
3837  throw std::runtime_error(
3838  "Query execution has been interrupted while performing " + work_type_str);
3839  }
3840  auto& result_rows = res.rs;
3841  result_rows->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
3842  const auto num_rows = result_rows->rowCount();
3843 
3844  if (0 == num_rows) {
3845  continue;
3846  }
3847 
3848  total_row_count += num_rows;
3849 
3850  size_t leaf_count = leafs_connector_->leafCount();
3851 
3852  // ensure that at least 1 row is processed per block up to a maximum of 65536 rows
3853  const size_t rows_per_block =
3854  std::max(std::min(num_rows / leaf_count, size_t(64 * 1024)), size_t(1));
3855 
3856  std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
3857 
3859 
3860  const int num_worker_threads = std::thread::hardware_concurrency();
3861 
3862  std::vector<size_t> thread_start_idx(num_worker_threads),
3863  thread_end_idx(num_worker_threads);
3864  bool can_go_parallel = !result_rows->isTruncated() && rows_per_block > 20000;
3865 
3866  std::atomic<size_t> crt_row_idx{0};
3867 
3868  auto do_work = [&result_rows, &value_converters, &crt_row_idx](
3869  const size_t idx,
3870  const size_t block_end,
3871  const size_t num_cols,
3872  const size_t thread_id,
3873  bool& stop_convert) {
3874  const auto result_row = result_rows->getRowAtNoTranslations(idx);
3875  if (!result_row.empty()) {
3876  size_t target_row = crt_row_idx.fetch_add(1);
3877  if (target_row >= block_end) {
3878  stop_convert = true;
3879  return;
3880  }
3881  for (unsigned int col = 0; col < num_cols; col++) {
3882  const auto& mapd_variant = result_row[col];
3883  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
3884  }
3885  }
3886  };
3887 
3888  auto convert_function = [&thread_start_idx,
3889  &thread_end_idx,
3890  &value_converters,
3891  &executor,
3892  &query_session,
3893  &work_type_str,
3894  &do_work](const int thread_id, const size_t block_end) {
3895  const int num_cols = value_converters.size();
3896  const size_t start = thread_start_idx[thread_id];
3897  const size_t end = thread_end_idx[thread_id];
3898  size_t idx = 0;
3899  bool stop_convert = false;
3901  size_t local_idx = 0;
3902  for (idx = start; idx < end; ++idx, ++local_idx) {
3903  if (UNLIKELY((local_idx & 0xFFFF) == 0 &&
3904  check_session_interrupted(query_session, executor))) {
3905  throw std::runtime_error(
3906  "Query execution has been interrupted while performing " +
3907  work_type_str);
3908  }
3909  do_work(idx, block_end, num_cols, thread_id, stop_convert);
3910  if (stop_convert) {
3911  break;
3912  }
3913  }
3914  } else {
3915  for (idx = start; idx < end; ++idx) {
3916  do_work(idx, block_end, num_cols, thread_id, stop_convert);
3917  if (stop_convert) {
3918  break;
3919  }
3920  }
3921  }
3922  thread_start_idx[thread_id] = idx;
3923  };
3924 
3925  auto single_threaded_value_converter =
3926  [&crt_row_idx, &value_converters, &result_rows](const size_t idx,
3927  const size_t block_end,
3928  const size_t num_cols,
3929  bool& stop_convert) {
3930  size_t target_row = crt_row_idx.fetch_add(1);
3931  if (target_row >= block_end) {
3932  stop_convert = true;
3933  return;
3934  }
3935  const auto result_row = result_rows->getNextRow(false, false);
3936  CHECK(!result_row.empty());
3937  for (unsigned int col = 0; col < num_cols; col++) {
3938  const auto& mapd_variant = result_row[col];
3939  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
3940  }
3941  };
3942 
3943  auto single_threaded_convert_function = [&value_converters,
3944  &thread_start_idx,
3945  &thread_end_idx,
3946  &executor,
3947  &query_session,
3948  &work_type_str,
3949  &single_threaded_value_converter](
3950  const int thread_id,
3951  const size_t block_end) {
3952  const int num_cols = value_converters.size();
3953  const size_t start = thread_start_idx[thread_id];
3954  const size_t end = thread_end_idx[thread_id];
3955  size_t idx = 0;
3956  bool stop_convert = false;
3958  size_t local_idx = 0;
3959  for (idx = start; idx < end; ++idx, ++local_idx) {
3960  if (UNLIKELY((local_idx & 0xFFFF) == 0 &&
3961  check_session_interrupted(query_session, executor))) {
3962  throw std::runtime_error(
3963  "Query execution has been interrupted while performing " +
3964  work_type_str);
3965  }
3966  single_threaded_value_converter(idx, block_end, num_cols, stop_convert);
3967  if (stop_convert) {
3968  break;
3969  }
3970  }
3971  } else {
3972  for (idx = start; idx < end; ++idx) {
3973  single_threaded_value_converter(idx, end, num_cols, stop_convert);
3974  if (stop_convert) {
3975  break;
3976  }
3977  }
3978  }
3979  thread_start_idx[thread_id] = idx;
3980  };
3981 
3982  if (can_go_parallel) {
3983  const size_t entry_count = result_rows->entryCount();
3984  for (size_t
3985  i = 0,
3986  start_entry = 0,
3987  stride = (entry_count + num_worker_threads - 1) / num_worker_threads;
3988  i < num_worker_threads && start_entry < entry_count;
3989  ++i, start_entry += stride) {
3990  const auto end_entry = std::min(start_entry + stride, entry_count);
3991  thread_start_idx[i] = start_entry;
3992  thread_end_idx[i] = end_entry;
3993  }
3994  } else {
3995  thread_start_idx[0] = 0;
3996  thread_end_idx[0] = result_rows->entryCount();
3997  }
3998 
3999  RenderGroupAnalyzerMap render_group_analyzer_map;
4000 
4001  for (size_t block_start = 0; block_start < num_rows;
4002  block_start += rows_per_block) {
4003  const auto num_rows_this_itr = block_start + rows_per_block < num_rows
4004  ? rows_per_block
4005  : num_rows - block_start;
4006  crt_row_idx = 0; // reset block tracker
4007  value_converters.clear();
4008  int colNum = 0;
4009  for (const auto targetDescriptor : target_column_descriptors) {
4010  auto sourceDataMetaInfo = res.targets_meta[colNum++];
4012  num_rows_this_itr,
4013  catalog,
4014  sourceDataMetaInfo,
4015  targetDescriptor,
4016  targetDescriptor->columnType,
4017  !targetDescriptor->columnType.get_notnull(),
4018  result_rows->getRowSetMemOwner()->getLiteralStringDictProxy(),
4020  sourceDataMetaInfo.get_type_info().is_dict_encoded_string()
4021  ? executor->getStringDictionaryProxy(
4022  sourceDataMetaInfo.get_type_info().get_comp_param(),
4023  result_rows->getRowSetMemOwner(),
4024  true)
4025  : nullptr,
4026  IS_GEO_POLY(targetDescriptor->columnType.get_type()) &&
4028  ? &render_group_analyzer_map
4029  : nullptr};
4030  auto converter = factory.create(param);
4031  value_converters.push_back(std::move(converter));
4032  }
4033 
4034  const auto translate_clock_begin = timer_start();
4035  if (can_go_parallel) {
4036  std::vector<std::future<void>> worker_threads;
4037  for (int i = 0; i < num_worker_threads; ++i) {
4038  worker_threads.push_back(
4039  std::async(std::launch::async, convert_function, i, num_rows_this_itr));
4040  }
4041 
4042  for (auto& child : worker_threads) {
4043  child.wait();
4044  }
4045  for (auto& child : worker_threads) {
4046  child.get();
4047  }
4048 
4049  } else {
4050  single_threaded_convert_function(0, num_rows_this_itr);
4051  }
4052 
4053  // finalize the insert data
4054  auto finalizer_func =
4055  [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
4056  targetValueConverter->finalizeDataBlocksForInsertData();
4057  };
4058 
4059  std::vector<std::future<void>> worker_threads;
4060  for (auto& converterPtr : value_converters) {
4061  worker_threads.push_back(
4062  std::async(std::launch::async, finalizer_func, converterPtr.get()));
4063  }
4064 
4065  for (auto& child : worker_threads) {
4066  child.wait();
4067  }
4068  for (auto& child : worker_threads) {
4069  child.get();
4070  }
4071 
4073  insert_data.databaseId = catalog.getCurrentDB().dbId;
4074  CHECK(td);
4075  insert_data.tableId = td->tableId;
4076  insert_data.numRows = num_rows_this_itr;
4077 
4078  for (int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
4080  check_session_interrupted(query_session, executor))) {
4081  throw std::runtime_error(
4082  "Query execution has been interrupted while performing " +
4083  work_type_str);
4084  }
4085  value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
4086  }
4087  total_target_value_translate_time_ms += timer_stop(translate_clock_begin);
4088 
4089  const auto data_load_clock_begin = timer_start();
4090  auto data_memory_holder =
4091  import_export::fill_missing_columns(&catalog, insert_data);
4092  insertDataLoader.insertData(*session, insert_data);
4093  total_data_load_time_ms += timer_stop(data_load_clock_begin);
4094  }
4095  }
4096  }
4097  } catch (...) {
4098  try {
4099  leafs_connector_->rollback(*session, td->tableId);
4100  } catch (std::exception& e) {
4101  LOG(ERROR) << "An error occurred during ITAS rollback attempt. Table id: "
4102  << td->tableId << ", Error: " << e.what();
4103  }
4104  throw;
4105  }
4106 
4107  int64_t total_time_ms = total_source_query_time_ms +
4108  total_target_value_translate_time_ms + total_data_load_time_ms;
4109 
4110  VLOG(1) << "CTAS/ITAS " << total_row_count << " rows loaded in " << total_time_ms
4111  << "ms (outer_frag_count=" << outer_frag_count
4112  << ", query_time=" << total_source_query_time_ms
4113  << "ms, translation_time=" << total_target_value_translate_time_ms
4114  << "ms, data_load_time=" << total_data_load_time_ms
4115  << "ms)\nquery: " << select_query_;
4116 
4117  if (!is_temporary) {
4118  leafs_connector_->checkpoint(*session, td->tableId);
4119  }
4120 }
void validate_non_foreign_table_write(const TableDescriptor *table_descriptor)
Definition: FsiUtils.h:22
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:405
HOST DEVICE int get_size() const
Definition: sqltypes.h:414
bool is_timestamp() const
Definition: sqltypes.h:1020
#define LOG(tag)
Definition: Logger.h:216
HOST DEVICE int get_scale() const
Definition: sqltypes.h:409
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:161
std::vector< std::unique_ptr< TypedImportBuffer > > fill_missing_columns(const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data)
Definition: Importer.cpp:6259
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:48
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:126
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:404
bool is_time() const
Definition: sqltypes.h:606
bool g_enable_string_functions
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:70
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:477
size_t numRows
a vector of column ids for the row(s) being inserted
Definition: Fragmenter.h:72
future< Result > async(Fn &&fn, Args &&...args)
std::unique_ptr< QueryConnector > leafs_connector_
Definition: ParserNode.h:1156
bool g_enable_assign_render_groups
QueryState & getQueryState()
Definition: QueryState.h:181
bool is_integer() const
Definition: sqltypes.h:602
specifies the content in-memory of a row in the column metadata table
bool check_session_interrupted(const QuerySessionId &query_session, Executor *executor)
Definition: ParserNode.cpp:102
#define UNLIKELY(x)
Definition: likely.h:25
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:412
bool table_is_temporary(const TableDescriptor *const td)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:406
std::string get_type_name() const
Definition: sqltypes.h:528
std::vector< std::unique_ptr< std::string > > column_list_
Definition: ParserNode.h:1159
ThreadId thread_id()
Definition: Logger.cpp:820
#define CHECK(condition)
Definition: Logger.h:222
bool is_geometry() const
Definition: sqltypes.h:612
bool g_cluster
The data to be inserted using the fragment manager.
Definition: Fragmenter.h:68
std::map< int, import_export::RenderGroupAnalyzer > RenderGroupAnalyzerMap
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:600
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:981
bool is_decimal() const
Definition: sqltypes.h:603
std::string columnName
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:88
bool is_date() const
Definition: sqltypes.h:1008
bool is_array() const
Definition: sqltypes.h:608
const std::string getQuerySubmittedTime() const
Definition: QueryState.cpp:105
#define VLOG(n)
Definition: Logger.h:316
Type timer_start()
Definition: measure.h:42
std::unique_ptr< TargetValueConverter > create(ConverterCreateParameter param)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:376
#define IS_GEO_POLY(T)
Definition: sqltypes.h:328

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::vector<std::unique_ptr<std::string> > Parser::InsertIntoTableAsSelectStmt::column_list_
protected

Definition at line 1159 of file ParserNode.h.

Referenced by InsertIntoTableAsSelectStmt(), and populateData().

std::unique_ptr<QueryConnector> Parser::InsertIntoTableAsSelectStmt::leafs_connector_

Definition at line 1156 of file ParserNode.h.

Referenced by Parser::CreateTableAsSelectStmt::execute(), and populateData().

std::string Parser::InsertIntoTableAsSelectStmt::select_query_
protected
std::string Parser::InsertIntoTableAsSelectStmt::table_name_
protected

The documentation for this class was generated from the following files: