OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Parser::InsertIntoTableAsSelectStmt Class Reference

#include <ParserNode.h>

+ Inheritance diagram for Parser::InsertIntoTableAsSelectStmt:
+ Collaboration diagram for Parser::InsertIntoTableAsSelectStmt:

Public Member Functions

 InsertIntoTableAsSelectStmt (const rapidjson::Value &payload)
 
 InsertIntoTableAsSelectStmt (const std::string *table_name, const std::string *select_query, std::list< std::string * > *c)
 
void populateData (QueryStateProxy, const TableDescriptor *td, bool validate_table, bool for_CTAS=false)
 
void execute (const Catalog_Namespace::SessionInfo &session, bool read_only_mode) override
 
std::string & get_table ()
 
std::string & get_select_query ()
 
- Public Member Functions inherited from Parser::DDLStmt
void setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)
 
- Public Member Functions inherited from Parser::Node
virtual ~Node ()
 

Public Attributes

std::unique_ptr< QueryConnectorleafs_connector_
 

Protected Attributes

std::vector< std::unique_ptr
< std::string > > 
column_list_
 
std::string table_name_
 
std::string select_query_
 

Detailed Description

Definition at line 1127 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::InsertIntoTableAsSelectStmt::InsertIntoTableAsSelectStmt ( const rapidjson::Value &  payload)

Definition at line 4063 of file ParserNode.cpp.

References CHECK, column_list_, json_str(), select_query_, and table_name_.

4064  {
4065  CHECK(payload.HasMember("name"));
4066  table_name_ = json_str(payload["name"]);
4067 
4068  CHECK(payload.HasMember("query"));
4069  select_query_ = json_str(payload["query"]);
4070 
4071  boost::replace_all(select_query_, "\n", " ");
4072  select_query_ = "(" + select_query_ + ")";
4073 
4074  if (payload.HasMember("columns")) {
4075  CHECK(payload["columns"].IsArray());
4076  for (auto& column : payload["columns"].GetArray()) {
4077  std::string s = json_str(column);
4078  column_list_.emplace_back(std::unique_ptr<std::string>(new std::string(s)));
4079  }
4080  }
4081 }
const std::string json_str(const rapidjson::Value &obj) noexcept
Definition: JsonAccessors.h:46
std::vector< std::unique_ptr< std::string > > column_list_
Definition: ParserNode.h:1160
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

Parser::InsertIntoTableAsSelectStmt::InsertIntoTableAsSelectStmt ( const std::string *  table_name,
const std::string *  select_query,
std::list< std::string * > *  c 
)
inline

Definition at line 1131 of file ParserNode.h.

References column_list_.

1134  : table_name_(*table_name), select_query_(*select_query) {
1135  if (c) {
1136  for (auto e : *c) {
1137  column_list_.emplace_back(e);
1138  }
1139  delete c;
1140  }
1141 
1142  delete table_name;
1143  delete select_query;
1144  }
std::vector< std::unique_ptr< std::string > > column_list_
Definition: ParserNode.h:1160

Member Function Documentation

void Parser::InsertIntoTableAsSelectStmt::execute ( const Catalog_Namespace::SessionInfo session,
bool  read_only_mode 
)
overridevirtual

Implements Parser::DDLStmt.

Reimplemented in Parser::CreateTableAsSelectStmt.

Definition at line 4651 of file ParserNode.cpp.

References Parser::anonymous_namespace{ParserNode.cpp}::acquire_query_table_locks(), Executor::clearExternalCaches(), query_state::QueryState::create(), legacylockmgr::getExecuteReadLock(), populateData(), select_query_, STDLOG, and table_name_.

Referenced by heavydb.cursor.Cursor::executemany().

4652  {
4653  if (read_only_mode) {
4654  throw std::runtime_error("INSERT INTO TABLE invalid in read only mode.");
4655  }
4656  auto session_copy = session;
4657  auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
4658  &session_copy, boost::null_deleter());
4659  auto query_state = query_state::QueryState::create(session_ptr, select_query_);
4660  auto stdlog = STDLOG(query_state);
4661  auto& catalog = session_ptr->getCatalog();
4662 
4663  const auto execute_read_lock = legacylockmgr::getExecuteReadLock();
4664 
4665  if (catalog.getMetadataForTable(table_name_) == nullptr) {
4666  throw std::runtime_error("ITAS failed: table " + table_name_ + " does not exist.");
4667  }
4668 
4669  auto locks = acquire_query_table_locks(
4670  catalog.name(), select_query_, query_state->createQueryStateProxy(), table_name_);
4671  const TableDescriptor* td = catalog.getMetadataForTable(table_name_);
4672 
4673  Executor::clearExternalCaches(true, td, catalog.getCurrentDB().dbId);
4674 
4675  try {
4676  populateData(query_state->createQueryStateProxy(), td, true, false);
4677  } catch (...) {
4678  throw;
4679  }
4680 }
auto getExecuteReadLock()
void populateData(QueryStateProxy, const TableDescriptor *td, bool validate_table, bool for_CTAS=false)
static std::shared_ptr< QueryState > create(ARGS &&...args)
Definition: QueryState.h:148
lockmgr::LockedTableDescriptors acquire_query_table_locks(const std::string &insert_table_db_name, const std::string &query_str, const QueryStateProxy &query_state_proxy, const std::optional< std::string > &insert_table_name={})
static void clearExternalCaches(bool for_update, const TableDescriptor *td, const int current_db_id)
Definition: Execute.h:438
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string& Parser::InsertIntoTableAsSelectStmt::get_select_query ( )
inline

Definition at line 1155 of file ParserNode.h.

References select_query_.

1155 { return select_query_; }
std::string& Parser::InsertIntoTableAsSelectStmt::get_table ( )
inline

Definition at line 1153 of file ParserNode.h.

References table_name_.

1153 { return table_name_; }
void Parser::InsertIntoTableAsSelectStmt::populateData ( QueryStateProxy  query_state_proxy,
const TableDescriptor td,
bool  validate_table,
bool  for_CTAS = false 
)

Definition at line 4083 of file ParserNode.cpp.

References threading_serial::async(), CHECK, Parser::check_session_interrupted(), column_list_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, TargetValueConverterFactory::create(), Fragmenter_Namespace::InsertData::databaseId, logger::ERROR, import_export::fill_missing_columns(), g_cluster, g_enable_non_kernel_time_query_interrupt, g_enable_string_functions, ResultSet::GeoTargetValue, SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), Parser::LocalQueryConnector::getColumnDescriptors(), query_state::QueryState::getConstSessionInfo(), Executor::getExecutor(), query_state::QueryState::getQuerySubmittedTime(), AccessPrivileges::INSERT_INTO_TABLE, Fragmenter_Namespace::InsertDataLoader::insertData(), SQLTypeInfo::is_array(), SQLTypeInfo::is_date(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_geometry(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), SQLTypeInfo::is_timestamp(), TableDescriptor::isView, leafs_connector_, LOG, Fragmenter_Namespace::InsertData::numRows, Parser::LocalQueryConnector::query(), run_benchmark_import::res, run_benchmark_import::result, select_query_, run_benchmark_import::start_time, table_is_temporary(), table_name_, TableDBObjectType, TableDescriptor::tableId, Fragmenter_Namespace::InsertData::tableId, logger::thread_id(), timer_start(), timer_stop(), Executor::UNITARY_EXECUTOR_ID, UNLIKELY, foreign_storage::validate_non_foreign_table_write(), and VLOG.

Referenced by execute(), and Parser::CreateTableAsSelectStmt::execute().

4086  {
4087  auto const session = query_state_proxy->getConstSessionInfo();
4088  auto& catalog = session->getCatalog();
4090  bool populate_table = false;
4091 
4092  if (leafs_connector_) {
4093  populate_table = true;
4094  } else {
4095  leafs_connector_ = std::make_unique<LocalQueryConnector>();
4096  if (!g_cluster) {
4097  populate_table = true;
4098  }
4099  }
4100 
4101  auto get_target_column_descriptors = [this, &catalog](const TableDescriptor* td) {
4102  std::vector<const ColumnDescriptor*> target_column_descriptors;
4103  if (column_list_.empty()) {
4104  auto list = catalog.getAllColumnMetadataForTable(td->tableId, false, false, false);
4105  target_column_descriptors = {std::begin(list), std::end(list)};
4106  } else {
4107  for (auto& c : column_list_) {
4108  const ColumnDescriptor* cd = catalog.getMetadataForColumn(td->tableId, *c);
4109  if (cd == nullptr) {
4110  throw std::runtime_error("Column " + *c + " does not exist.");
4111  }
4112  target_column_descriptors.push_back(cd);
4113  }
4114  }
4115 
4116  return target_column_descriptors;
4117  };
4118 
4119  bool is_temporary = table_is_temporary(td);
4120 
4121  if (validate_table) {
4122  // check access privileges
4123  if (!td) {
4124  throw std::runtime_error("Table " + table_name_ + " does not exist.");
4125  }
4126  if (td->isView) {
4127  throw std::runtime_error("Insert to views is not supported yet.");
4128  }
4129 
4130  if (!session->checkDBAccessPrivileges(DBObjectType::TableDBObjectType,
4132  table_name_)) {
4133  throw std::runtime_error("User has no insert privileges on " + table_name_ + ".");
4134  }
4135 
4136  // only validate the select query so we get the target types
4137  // correctly, but do not populate the result set
4138  LocalQueryConnector local_connector;
4139  auto result = local_connector.query(query_state_proxy, select_query_, {}, true, true);
4140  auto source_column_descriptors = local_connector.getColumnDescriptors(result, false);
4141 
4142  std::vector<const ColumnDescriptor*> target_column_descriptors =
4143  get_target_column_descriptors(td);
4144 
4145  if (source_column_descriptors.size() != target_column_descriptors.size()) {
4146  throw std::runtime_error("The number of source and target columns does not match.");
4147  }
4148 
4149  for (int i = 0; i < source_column_descriptors.size(); i++) {
4150  const ColumnDescriptor* source_cd =
4151  &(*std::next(source_column_descriptors.begin(), i));
4152  const ColumnDescriptor* target_cd = target_column_descriptors.at(i);
4153 
4154  if (source_cd->columnType.get_type() != target_cd->columnType.get_type()) {
4155  auto type_cannot_be_cast = [](const auto& col_type) {
4156  return (col_type.is_time() || col_type.is_geometry() || col_type.is_array() ||
4157  col_type.is_boolean());
4158  };
4159 
4160  if (type_cannot_be_cast(source_cd->columnType) ||
4161  type_cannot_be_cast(target_cd->columnType)) {
4162  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4163  source_cd->columnType.get_type_name() +
4164  "' and target '" + target_cd->columnName + " " +
4165  target_cd->columnType.get_type_name() +
4166  "' column types do not match.");
4167  }
4168  }
4169  if (source_cd->columnType.is_array()) {
4170  if (source_cd->columnType.get_subtype() != target_cd->columnType.get_subtype()) {
4171  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4172  source_cd->columnType.get_type_name() +
4173  "' and target '" + target_cd->columnName + " " +
4174  target_cd->columnType.get_type_name() +
4175  "' array column element types do not match.");
4176  }
4177  }
4178 
4179  if (target_cd->columnType.is_string() && !source_cd->columnType.is_string()) {
4180  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4181  source_cd->columnType.get_type_name() +
4182  "' and target '" + target_cd->columnName + " " +
4183  target_cd->columnType.get_type_name() +
4184  "' column types do not match.");
4185  }
4186 
4187  if (source_cd->columnType.is_decimal() ||
4188  source_cd->columnType.get_elem_type().is_decimal()) {
4189  SQLTypeInfo sourceType = source_cd->columnType;
4190  SQLTypeInfo targetType = target_cd->columnType;
4191 
4192  if (source_cd->columnType.is_array()) {
4193  sourceType = source_cd->columnType.get_elem_type();
4194  targetType = target_cd->columnType.get_elem_type();
4195  }
4196 
4197  if (sourceType.get_scale() != targetType.get_scale()) {
4198  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4199  source_cd->columnType.get_type_name() +
4200  "' and target '" + target_cd->columnName + " " +
4201  target_cd->columnType.get_type_name() +
4202  "' decimal columns scales do not match.");
4203  }
4204  }
4205 
4206  if (source_cd->columnType.is_string()) {
4207  if (!target_cd->columnType.is_string()) {
4208  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4209  source_cd->columnType.get_type_name() +
4210  "' and target '" + target_cd->columnName + " " +
4211  target_cd->columnType.get_type_name() +
4212  "' column types do not match.");
4213  }
4214  if (source_cd->columnType.get_compression() !=
4215  target_cd->columnType.get_compression()) {
4216  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4217  source_cd->columnType.get_type_name() +
4218  "' and target '" + target_cd->columnName + " " +
4219  target_cd->columnType.get_type_name() +
4220  "' columns string encodings do not match.");
4221  }
4222  }
4223 
4224  if (source_cd->columnType.is_timestamp() && target_cd->columnType.is_timestamp()) {
4225  if (source_cd->columnType.get_dimension() !=
4226  target_cd->columnType.get_dimension()) {
4227  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4228  source_cd->columnType.get_type_name() +
4229  "' and target '" + target_cd->columnName + " " +
4230  target_cd->columnType.get_type_name() +
4231  "' timestamp column precisions do not match.");
4232  }
4233  }
4234 
4235  if (!source_cd->columnType.is_string() && !source_cd->columnType.is_geometry() &&
4236  !source_cd->columnType.is_integer() && !source_cd->columnType.is_decimal() &&
4237  !source_cd->columnType.is_date() && !source_cd->columnType.is_time() &&
4238  !source_cd->columnType.is_timestamp() &&
4239  source_cd->columnType.get_size() > target_cd->columnType.get_size()) {
4240  throw std::runtime_error("Source '" + source_cd->columnName + " " +
4241  source_cd->columnType.get_type_name() +
4242  "' and target '" + target_cd->columnName + " " +
4243  target_cd->columnType.get_type_name() +
4244  "' column encoding sizes do not match.");
4245  }
4246  }
4247  }
4248 
4249  if (!populate_table) {
4250  return;
4251  }
4252 
4253  int64_t total_row_count = 0;
4254  int64_t total_source_query_time_ms = 0;
4255  int64_t total_target_value_translate_time_ms = 0;
4256  int64_t total_data_load_time_ms = 0;
4257 
4259  auto target_column_descriptors = get_target_column_descriptors(td);
4260  auto outer_frag_count =
4261  leafs_connector_->getOuterFragmentCount(query_state_proxy, select_query_);
4262 
4263  size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
4264  auto query_session = session ? session->get_session_id() : "";
4266  std::string work_type_str = for_CTAS ? "CTAS" : "ITAS";
4267  try {
4268  for (size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
4269  std::vector<size_t> allowed_outer_fragment_indices;
4270 
4271  if (outer_frag_count) {
4272  allowed_outer_fragment_indices.push_back(outer_frag_idx);
4273  }
4274 
4275  const auto query_clock_begin = timer_start();
4276  std::vector<AggregatedResult> query_results =
4277  leafs_connector_->query(query_state_proxy,
4278  select_query_,
4279  allowed_outer_fragment_indices,
4281  total_source_query_time_ms += timer_stop(query_clock_begin);
4282 
4283  auto start_time = query_state_proxy->getQuerySubmittedTime();
4284  auto query_str = "INSERT_DATA for " + work_type_str;
4286  // In the clean-up phase of the query execution for collecting aggregated result
4287  // of SELECT query, we remove its query session info, so we need to enroll the
4288  // session info again
4289  executor->enrollQuerySession(query_session,
4290  query_str,
4291  start_time,
4293  QuerySessionStatus::QueryStatus::RUNNING_IMPORTER);
4294  }
4295 
4296  ScopeGuard clearInterruptStatus = [executor, &query_session, &start_time] {
4297  // this data population is non-kernel operation, so we manually cleanup
4298  // the query session info in the cleanup phase
4300  executor->clearQuerySessionStatus(query_session, start_time);
4301  }
4302  };
4303 
4304  for (auto& res : query_results) {
4305  if (UNLIKELY(check_session_interrupted(query_session, executor))) {
4306  throw std::runtime_error(
4307  "Query execution has been interrupted while performing " + work_type_str);
4308  }
4309  auto& result_rows = res.rs;
4310  result_rows->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
4311  const auto num_rows = result_rows->rowCount();
4312 
4313  if (0 == num_rows) {
4314  continue;
4315  }
4316 
4317  total_row_count += num_rows;
4318 
4319  size_t leaf_count = leafs_connector_->leafCount();
4320 
4321  // ensure that at least 1 row is processed per block up to a maximum of 65536 rows
4322  const size_t rows_per_block =
4323  std::max(std::min(num_rows / leaf_count, size_t(64 * 1024)), size_t(1));
4324 
4325  std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
4326 
4328 
4329  const int num_worker_threads = std::thread::hardware_concurrency();
4330 
4331  std::vector<size_t> thread_start_idx(num_worker_threads),
4332  thread_end_idx(num_worker_threads);
4333  bool can_go_parallel = !result_rows->isTruncated() && rows_per_block > 20000;
4334 
4335  std::atomic<size_t> crt_row_idx{0};
4336 
4337  auto do_work = [&result_rows, &value_converters, &crt_row_idx](
4338  const size_t idx,
4339  const size_t block_end,
4340  const size_t num_cols,
4341  const size_t thread_id,
4342  bool& stop_convert) {
4343  const auto result_row = result_rows->getRowAtNoTranslations(idx);
4344  if (!result_row.empty()) {
4345  size_t target_row = crt_row_idx.fetch_add(1);
4346  if (target_row >= block_end) {
4347  stop_convert = true;
4348  return;
4349  }
4350  for (unsigned int col = 0; col < num_cols; col++) {
4351  const auto& mapd_variant = result_row[col];
4352  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
4353  }
4354  }
4355  };
4356 
4357  auto convert_function = [&thread_start_idx,
4358  &thread_end_idx,
4359  &value_converters,
4360  &executor,
4361  &query_session,
4362  &work_type_str,
4363  &do_work](const int thread_id, const size_t block_end) {
4364  const int num_cols = value_converters.size();
4365  const size_t start = thread_start_idx[thread_id];
4366  const size_t end = thread_end_idx[thread_id];
4367  size_t idx = 0;
4368  bool stop_convert = false;
4370  size_t local_idx = 0;
4371  for (idx = start; idx < end; ++idx, ++local_idx) {
4372  if (UNLIKELY((local_idx & 0xFFFF) == 0 &&
4373  check_session_interrupted(query_session, executor))) {
4374  throw std::runtime_error(
4375  "Query execution has been interrupted while performing " +
4376  work_type_str);
4377  }
4378  do_work(idx, block_end, num_cols, thread_id, stop_convert);
4379  if (stop_convert) {
4380  break;
4381  }
4382  }
4383  } else {
4384  for (idx = start; idx < end; ++idx) {
4385  do_work(idx, block_end, num_cols, thread_id, stop_convert);
4386  if (stop_convert) {
4387  break;
4388  }
4389  }
4390  }
4391  thread_start_idx[thread_id] = idx;
4392  };
4393 
4394  auto single_threaded_value_converter =
4395  [&crt_row_idx, &value_converters, &result_rows](const size_t idx,
4396  const size_t block_end,
4397  const size_t num_cols,
4398  bool& stop_convert) {
4399  size_t target_row = crt_row_idx.fetch_add(1);
4400  if (target_row >= block_end) {
4401  stop_convert = true;
4402  return;
4403  }
4404  const auto result_row = result_rows->getNextRow(false, false);
4405  CHECK(!result_row.empty());
4406  for (unsigned int col = 0; col < num_cols; col++) {
4407  const auto& mapd_variant = result_row[col];
4408  value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
4409  }
4410  };
4411 
4412  auto single_threaded_convert_function = [&value_converters,
4413  &thread_start_idx,
4414  &thread_end_idx,
4415  &executor,
4416  &query_session,
4417  &work_type_str,
4418  &single_threaded_value_converter](
4419  const int thread_id,
4420  const size_t block_end) {
4421  const int num_cols = value_converters.size();
4422  const size_t start = thread_start_idx[thread_id];
4423  const size_t end = thread_end_idx[thread_id];
4424  size_t idx = 0;
4425  bool stop_convert = false;
4427  size_t local_idx = 0;
4428  for (idx = start; idx < end; ++idx, ++local_idx) {
4429  if (UNLIKELY((local_idx & 0xFFFF) == 0 &&
4430  check_session_interrupted(query_session, executor))) {
4431  throw std::runtime_error(
4432  "Query execution has been interrupted while performing " +
4433  work_type_str);
4434  }
4435  single_threaded_value_converter(idx, block_end, num_cols, stop_convert);
4436  if (stop_convert) {
4437  break;
4438  }
4439  }
4440  } else {
4441  for (idx = start; idx < end; ++idx) {
4442  single_threaded_value_converter(idx, end, num_cols, stop_convert);
4443  if (stop_convert) {
4444  break;
4445  }
4446  }
4447  }
4448  thread_start_idx[thread_id] = idx;
4449  };
4450 
4451  if (can_go_parallel) {
4452  const size_t entry_count = result_rows->entryCount();
4453  for (size_t
4454  i = 0,
4455  start_entry = 0,
4456  stride = (entry_count + num_worker_threads - 1) / num_worker_threads;
4457  i < num_worker_threads && start_entry < entry_count;
4458  ++i, start_entry += stride) {
4459  const auto end_entry = std::min(start_entry + stride, entry_count);
4460  thread_start_idx[i] = start_entry;
4461  thread_end_idx[i] = end_entry;
4462  }
4463  } else {
4464  thread_start_idx[0] = 0;
4465  thread_end_idx[0] = result_rows->entryCount();
4466  }
4467 
4468  for (size_t block_start = 0; block_start < num_rows;
4469  block_start += rows_per_block) {
4470  const auto num_rows_this_itr = block_start + rows_per_block < num_rows
4471  ? rows_per_block
4472  : num_rows - block_start;
4473  crt_row_idx = 0; // reset block tracker
4474  value_converters.clear();
4475  int colNum = 0;
4476  for (const auto targetDescriptor : target_column_descriptors) {
4477  auto sourceDataMetaInfo = res.targets_meta[colNum++];
4479  num_rows_this_itr,
4480  sourceDataMetaInfo,
4481  targetDescriptor,
4482  catalog,
4483  targetDescriptor->columnType,
4484  !targetDescriptor->columnType.get_notnull(),
4485  result_rows->getRowSetMemOwner()->getLiteralStringDictProxy(),
4487  sourceDataMetaInfo.get_type_info().is_dict_encoded_string()
4488  ? executor->getStringDictionaryProxy(
4489  sourceDataMetaInfo.get_type_info().getStringDictKey(),
4490  result_rows->getRowSetMemOwner(),
4491  true)
4492  : nullptr};
4493  auto converter = factory.create(param);
4494  value_converters.push_back(std::move(converter));
4495  }
4496 
4497  const auto translate_clock_begin = timer_start();
4498  if (can_go_parallel) {
4499  std::vector<std::future<void>> worker_threads;
4500  for (int i = 0; i < num_worker_threads; ++i) {
4501  worker_threads.push_back(
4502  std::async(std::launch::async, convert_function, i, num_rows_this_itr));
4503  }
4504 
4505  for (auto& child : worker_threads) {
4506  child.wait();
4507  }
4508  for (auto& child : worker_threads) {
4509  child.get();
4510  }
4511 
4512  } else {
4513  single_threaded_convert_function(0, num_rows_this_itr);
4514  }
4515 
4516  // finalize the insert data
4517  auto finalizer_func =
4518  [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
4519  targetValueConverter->finalizeDataBlocksForInsertData();
4520  };
4521 
4522  std::vector<std::future<void>> worker_threads;
4523  for (auto& converterPtr : value_converters) {
4524  worker_threads.push_back(
4525  std::async(std::launch::async, finalizer_func, converterPtr.get()));
4526  }
4527 
4528  for (auto& child : worker_threads) {
4529  child.wait();
4530  }
4531  for (auto& child : worker_threads) {
4532  child.get();
4533  }
4534 
4536  insert_data.databaseId = catalog.getCurrentDB().dbId;
4537  CHECK(td);
4538  insert_data.tableId = td->tableId;
4539  insert_data.numRows = num_rows_this_itr;
4540 
4541  for (int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
4543  check_session_interrupted(query_session, executor))) {
4544  throw std::runtime_error(
4545  "Query execution has been interrupted while performing " +
4546  work_type_str);
4547  }
4548  value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
4549  }
4550  total_target_value_translate_time_ms += timer_stop(translate_clock_begin);
4551 
4552  const auto data_load_clock_begin = timer_start();
4553  auto data_memory_holder =
4554  import_export::fill_missing_columns(&catalog, insert_data);
4555  insertDataLoader.insertData(*session, insert_data);
4556  total_data_load_time_ms += timer_stop(data_load_clock_begin);
4557  }
4558  }
4559  }
4560  } catch (...) {
4561  try {
4562  leafs_connector_->rollback(*session, td->tableId);
4563  } catch (std::exception& e) {
4564  LOG(ERROR) << "An error occurred during ITAS rollback attempt. Table id: "
4565  << td->tableId << ", Error: " << e.what();
4566  }
4567  throw;
4568  }
4569 
4570  int64_t total_time_ms = total_source_query_time_ms +
4571  total_target_value_translate_time_ms + total_data_load_time_ms;
4572 
4573  VLOG(1) << "CTAS/ITAS " << total_row_count << " rows loaded in " << total_time_ms
4574  << "ms (outer_frag_count=" << outer_frag_count
4575  << ", query_time=" << total_source_query_time_ms
4576  << "ms, translation_time=" << total_target_value_translate_time_ms
4577  << "ms, data_load_time=" << total_data_load_time_ms
4578  << "ms)\nquery: " << select_query_;
4579 
4580  if (!is_temporary) {
4581  leafs_connector_->checkpoint(*session, td->tableId);
4582  }
4583 }
void validate_non_foreign_table_write(const TableDescriptor *table_descriptor)
Definition: FsiUtils.h:22
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
bool is_timestamp() const
Definition: sqltypes.h:1044
#define LOG(tag)
Definition: Logger.h:285
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:161
std::vector< std::unique_ptr< TypedImportBuffer > > fill_missing_columns(const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data)
Definition: Importer.cpp:6141
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:48
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:134
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
bool is_time() const
Definition: sqltypes.h:577
bool g_enable_string_functions
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:70
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
size_t numRows
a vector of column ids for the row(s) being inserted
Definition: Fragmenter.h:72
future< Result > async(Fn &&fn, Args &&...args)
std::unique_ptr< QueryConnector > leafs_connector_
Definition: ParserNode.h:1157
bool is_integer() const
Definition: sqltypes.h:565
specifies the content in-memory of a row in the column metadata table
bool check_session_interrupted(const QuerySessionId &query_session, Executor *executor)
Definition: ParserNode.cpp:106
#define UNLIKELY(x)
Definition: likely.h:25
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
bool table_is_temporary(const TableDescriptor *const td)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
std::string get_type_name() const
Definition: sqltypes.h:482
std::vector< std::unique_ptr< std::string > > column_list_
Definition: ParserNode.h:1160
ThreadId thread_id()
Definition: Logger.cpp:877
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:595
bool g_cluster
The data to be inserted using the fragment manager.
Definition: Fragmenter.h:68
SQLTypeInfo columnType
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
bool is_string() const
Definition: sqltypes.h:559
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:975
bool is_decimal() const
Definition: sqltypes.h:568
std::string columnName
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:84
bool is_date() const
Definition: sqltypes.h:1026
bool is_array() const
Definition: sqltypes.h:583
const std::string getQuerySubmittedTime() const
Definition: QueryState.cpp:101
#define VLOG(n)
Definition: Logger.h:388
Type timer_start()
Definition: measure.h:42
std::unique_ptr< TargetValueConverter > create(ConverterCreateParameter param)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::vector<std::unique_ptr<std::string> > Parser::InsertIntoTableAsSelectStmt::column_list_
protected

Definition at line 1160 of file ParserNode.h.

Referenced by InsertIntoTableAsSelectStmt(), and populateData().

std::unique_ptr<QueryConnector> Parser::InsertIntoTableAsSelectStmt::leafs_connector_

Definition at line 1157 of file ParserNode.h.

Referenced by Parser::CreateTableAsSelectStmt::execute(), and populateData().

std::string Parser::InsertIntoTableAsSelectStmt::select_query_
protected
std::string Parser::InsertIntoTableAsSelectStmt::table_name_
protected

The documentation for this class was generated from the following files: