4088 auto& catalog = session->getCatalog();
4090 bool populate_table =
false;
4093 populate_table =
true;
4097 populate_table =
true;
4101 auto get_target_column_descriptors = [
this, &catalog](
const TableDescriptor* td) {
4102 std::vector<const ColumnDescriptor*> target_column_descriptors;
4104 auto list = catalog.getAllColumnMetadataForTable(td->
tableId,
false,
false,
false);
4105 target_column_descriptors = {std::begin(list), std::end(list)};
4109 if (cd ==
nullptr) {
4110 throw std::runtime_error(
"Column " + *c +
" does not exist.");
4112 target_column_descriptors.push_back(cd);
4116 return target_column_descriptors;
4121 if (validate_table) {
4124 throw std::runtime_error(
"Table " +
table_name_ +
" does not exist.");
4127 throw std::runtime_error(
"Insert to views is not supported yet.");
4133 throw std::runtime_error(
"User has no insert privileges on " +
table_name_ +
".");
4138 LocalQueryConnector local_connector;
4140 auto source_column_descriptors = local_connector.getColumnDescriptors(
result,
false);
4142 std::vector<const ColumnDescriptor*> target_column_descriptors =
4143 get_target_column_descriptors(td);
4145 if (source_column_descriptors.size() != target_column_descriptors.size()) {
4146 throw std::runtime_error(
"The number of source and target columns does not match.");
4149 for (
int i = 0; i < source_column_descriptors.size(); i++) {
4151 &(*std::next(source_column_descriptors.begin(), i));
4155 auto type_cannot_be_cast = [](
const auto& col_type) {
4156 return (col_type.is_time() || col_type.is_geometry() || col_type.is_array() ||
4157 col_type.is_boolean());
4160 if (type_cannot_be_cast(source_cd->
columnType) ||
4161 type_cannot_be_cast(target_cd->
columnType)) {
4162 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4164 "' and target '" + target_cd->
columnName +
" " +
4166 "' column types do not match.");
4171 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4173 "' and target '" + target_cd->
columnName +
" " +
4175 "' array column element types do not match.");
4180 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4182 "' and target '" + target_cd->
columnName +
" " +
4184 "' column types do not match.");
4198 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4200 "' and target '" + target_cd->
columnName +
" " +
4202 "' decimal columns scales do not match.");
4208 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4210 "' and target '" + target_cd->
columnName +
" " +
4212 "' column types do not match.");
4216 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4218 "' and target '" + target_cd->
columnName +
" " +
4220 "' columns string encodings do not match.");
4227 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4229 "' and target '" + target_cd->
columnName +
" " +
4231 "' timestamp column precisions do not match.");
4240 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
4242 "' and target '" + target_cd->
columnName +
" " +
4244 "' column encoding sizes do not match.");
4249 if (!populate_table) {
4253 int64_t total_row_count = 0;
4254 int64_t total_source_query_time_ms = 0;
4255 int64_t total_target_value_translate_time_ms = 0;
4256 int64_t total_data_load_time_ms = 0;
4259 auto target_column_descriptors = get_target_column_descriptors(td);
4260 auto outer_frag_count =
4263 size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
4264 auto query_session = session ? session->get_session_id() :
"";
4266 std::string work_type_str = for_CTAS ?
"CTAS" :
"ITAS";
4268 for (
size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
4269 std::vector<size_t> allowed_outer_fragment_indices;
4271 if (outer_frag_count) {
4272 allowed_outer_fragment_indices.push_back(outer_frag_idx);
4276 std::vector<AggregatedResult> query_results =
4279 allowed_outer_fragment_indices,
4281 total_source_query_time_ms +=
timer_stop(query_clock_begin);
4284 auto query_str =
"INSERT_DATA for " + work_type_str;
4289 executor->enrollQuerySession(query_session,
4293 QuerySessionStatus::QueryStatus::RUNNING_IMPORTER);
4300 executor->clearQuerySessionStatus(query_session,
start_time);
4304 for (
auto&
res : query_results) {
4306 throw std::runtime_error(
4307 "Query execution has been interrupted while performing " + work_type_str);
4309 auto& result_rows =
res.rs;
4311 const auto num_rows = result_rows->rowCount();
4313 if (0 == num_rows) {
4317 total_row_count += num_rows;
4322 const size_t rows_per_block =
4323 std::max(std::min(num_rows / leaf_count,
size_t(64 * 1024)),
size_t(1));
4325 std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
4329 const int num_worker_threads = std::thread::hardware_concurrency();
4331 std::vector<size_t> thread_start_idx(num_worker_threads),
4332 thread_end_idx(num_worker_threads);
4333 bool can_go_parallel = !result_rows->isTruncated() && rows_per_block > 20000;
4335 std::atomic<size_t> crt_row_idx{0};
4337 auto do_work = [&result_rows, &value_converters, &crt_row_idx](
4339 const size_t block_end,
4340 const size_t num_cols,
4342 bool& stop_convert) {
4343 const auto result_row = result_rows->getRowAtNoTranslations(idx);
4344 if (!result_row.empty()) {
4345 size_t target_row = crt_row_idx.fetch_add(1);
4346 if (target_row >= block_end) {
4347 stop_convert =
true;
4350 for (
unsigned int col = 0; col < num_cols; col++) {
4351 const auto& mapd_variant = result_row[col];
4352 value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
4357 auto convert_function = [&thread_start_idx,
4363 &do_work](
const int thread_id,
const size_t block_end) {
4364 const int num_cols = value_converters.size();
4365 const size_t start = thread_start_idx[
thread_id];
4366 const size_t end = thread_end_idx[
thread_id];
4368 bool stop_convert =
false;
4370 size_t local_idx = 0;
4371 for (idx = start; idx < end; ++idx, ++local_idx) {
4372 if (
UNLIKELY((local_idx & 0xFFFF) == 0 &&
4374 throw std::runtime_error(
4375 "Query execution has been interrupted while performing " +
4378 do_work(idx, block_end, num_cols, thread_id, stop_convert);
4384 for (idx = start; idx < end; ++idx) {
4385 do_work(idx, block_end, num_cols, thread_id, stop_convert);
4394 auto single_threaded_value_converter =
4395 [&crt_row_idx, &value_converters, &result_rows](
const size_t idx,
4396 const size_t block_end,
4397 const size_t num_cols,
4398 bool& stop_convert) {
4399 size_t target_row = crt_row_idx.fetch_add(1);
4400 if (target_row >= block_end) {
4401 stop_convert =
true;
4404 const auto result_row = result_rows->getNextRow(
false,
false);
4405 CHECK(!result_row.empty());
4406 for (
unsigned int col = 0; col < num_cols; col++) {
4407 const auto& mapd_variant = result_row[col];
4408 value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
4412 auto single_threaded_convert_function = [&value_converters,
4418 &single_threaded_value_converter](
4420 const size_t block_end) {
4421 const int num_cols = value_converters.size();
4422 const size_t start = thread_start_idx[
thread_id];
4423 const size_t end = thread_end_idx[
thread_id];
4425 bool stop_convert =
false;
4427 size_t local_idx = 0;
4428 for (idx = start; idx < end; ++idx, ++local_idx) {
4429 if (
UNLIKELY((local_idx & 0xFFFF) == 0 &&
4431 throw std::runtime_error(
4432 "Query execution has been interrupted while performing " +
4435 single_threaded_value_converter(idx, block_end, num_cols, stop_convert);
4441 for (idx = start; idx < end; ++idx) {
4442 single_threaded_value_converter(idx, end, num_cols, stop_convert);
4451 if (can_go_parallel) {
4452 const size_t entry_count = result_rows->entryCount();
4456 stride = (entry_count + num_worker_threads - 1) / num_worker_threads;
4457 i < num_worker_threads && start_entry < entry_count;
4458 ++i, start_entry += stride) {
4459 const auto end_entry = std::min(start_entry + stride, entry_count);
4460 thread_start_idx[i] = start_entry;
4461 thread_end_idx[i] = end_entry;
4464 thread_start_idx[0] = 0;
4465 thread_end_idx[0] = result_rows->entryCount();
4468 for (
size_t block_start = 0; block_start < num_rows;
4469 block_start += rows_per_block) {
4470 const auto num_rows_this_itr = block_start + rows_per_block < num_rows
4472 : num_rows - block_start;
4474 value_converters.clear();
4476 for (
const auto targetDescriptor : target_column_descriptors) {
4477 auto sourceDataMetaInfo =
res.targets_meta[colNum++];
4483 targetDescriptor->columnType,
4484 !targetDescriptor->columnType.get_notnull(),
4485 result_rows->getRowSetMemOwner()->getLiteralStringDictProxy(),
4487 sourceDataMetaInfo.get_type_info().is_dict_encoded_string()
4488 ? executor->getStringDictionaryProxy(
4489 sourceDataMetaInfo.get_type_info().getStringDictKey(),
4490 result_rows->getRowSetMemOwner(),
4493 auto converter = factory.
create(param);
4494 value_converters.push_back(std::move(converter));
4498 if (can_go_parallel) {
4499 std::vector<std::future<void>> worker_threads;
4500 for (
int i = 0; i < num_worker_threads; ++i) {
4501 worker_threads.push_back(
4505 for (
auto& child : worker_threads) {
4508 for (
auto& child : worker_threads) {
4513 single_threaded_convert_function(0, num_rows_this_itr);
4517 auto finalizer_func =
4518 [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
4519 targetValueConverter->finalizeDataBlocksForInsertData();
4522 std::vector<std::future<void>> worker_threads;
4523 for (
auto& converterPtr : value_converters) {
4524 worker_threads.push_back(
4528 for (
auto& child : worker_threads) {
4531 for (
auto& child : worker_threads) {
4536 insert_data.
databaseId = catalog.getCurrentDB().dbId;
4539 insert_data.
numRows = num_rows_this_itr;
4541 for (
int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
4544 throw std::runtime_error(
4545 "Query execution has been interrupted while performing " +
4548 value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
4550 total_target_value_translate_time_ms +=
timer_stop(translate_clock_begin);
4553 auto data_memory_holder =
4555 insertDataLoader.insertData(*session, insert_data);
4556 total_data_load_time_ms +=
timer_stop(data_load_clock_begin);
4563 }
catch (std::exception& e) {
4564 LOG(
ERROR) <<
"An error occurred during ITAS rollback attempt. Table id: "
4565 << td->
tableId <<
", Error: " << e.what();
4570 int64_t total_time_ms = total_source_query_time_ms +
4571 total_target_value_translate_time_ms + total_data_load_time_ms;
4573 VLOG(1) <<
"CTAS/ITAS " << total_row_count <<
" rows loaded in " << total_time_ms
4574 <<
"ms (outer_frag_count=" << outer_frag_count
4575 <<
", query_time=" << total_source_query_time_ms
4576 <<
"ms, translation_time=" << total_target_value_translate_time_ms
4577 <<
"ms, data_load_time=" << total_data_load_time_ms
4580 if (!is_temporary) {
void validate_non_foreign_table_write(const TableDescriptor *table_descriptor)
HOST DEVICE SQLTypes get_subtype() const
HOST DEVICE int get_size() const
bool is_timestamp() const
HOST DEVICE int get_scale() const
static const AccessPrivileges INSERT_INTO_TABLE
std::vector< std::unique_ptr< TypedImportBuffer > > fill_missing_columns(const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data)
TypeR::rep timer_stop(Type clock_begin)
bool g_enable_non_kernel_time_query_interrupt
HOST DEVICE SQLTypes get_type() const
bool g_enable_string_functions
int tableId
identifies the database into which the data is being inserted
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
size_t numRows
a vector of column ids for the row(s) being inserted
future< Result > async(Fn &&fn, Args &&...args)
std::unique_ptr< QueryConnector > leafs_connector_
specifies the content in-memory of a row in the column metadata table
bool check_session_interrupted(const QuerySessionId &query_session, Executor *executor)
HOST DEVICE EncodingType get_compression() const
bool table_is_temporary(const TableDescriptor *const td)
HOST DEVICE int get_dimension() const
std::string select_query_
std::string get_type_name() const
std::vector< std::unique_ptr< std::string > > column_list_
The data to be inserted using the fragment manager.
static constexpr ExecutorId UNITARY_EXECUTOR_ID
SQLTypeInfo get_elem_type() const
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
const std::string getQuerySubmittedTime() const
std::unique_ptr< TargetValueConverter > create(ConverterCreateParameter param)