3619 auto& catalog = session->getCatalog();
3621 bool populate_table =
false;
3624 populate_table =
true;
3628 populate_table =
true;
3632 auto get_target_column_descriptors = [
this, &catalog](
const TableDescriptor* td) {
3633 std::vector<const ColumnDescriptor*> target_column_descriptors;
3635 auto list = catalog.getAllColumnMetadataForTable(td->
tableId,
false,
false,
false);
3636 target_column_descriptors = {std::begin(list), std::end(list)};
3640 if (cd ==
nullptr) {
3641 throw std::runtime_error(
"Column " + *c +
" does not exist.");
3643 target_column_descriptors.push_back(cd);
3647 return target_column_descriptors;
3652 if (validate_table) {
3655 throw std::runtime_error(
"Table " +
table_name_ +
" does not exist.");
3658 throw std::runtime_error(
"Insert to views is not supported yet.");
3664 throw std::runtime_error(
"User has no insert privileges on " +
table_name_ +
".");
3669 LocalQueryConnector local_connector;
3671 auto source_column_descriptors = local_connector.getColumnDescriptors(
result,
false);
3673 std::vector<const ColumnDescriptor*> target_column_descriptors =
3674 get_target_column_descriptors(td);
3676 if (source_column_descriptors.size() != target_column_descriptors.size()) {
3677 throw std::runtime_error(
"The number of source and target columns does not match.");
3680 for (
int i = 0; i < source_column_descriptors.size(); i++) {
3682 &(*std::next(source_column_descriptors.begin(), i));
3686 auto type_cannot_be_cast = [](
const auto& col_type) {
3687 return (col_type.is_time() || col_type.is_geometry() || col_type.is_array() ||
3688 col_type.is_boolean());
3691 if (type_cannot_be_cast(source_cd->
columnType) ||
3692 type_cannot_be_cast(target_cd->
columnType)) {
3693 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3695 "' and target '" + target_cd->
columnName +
" " +
3697 "' column types do not match.");
3702 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3704 "' and target '" + target_cd->
columnName +
" " +
3706 "' array column element types do not match.");
3711 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3713 "' and target '" + target_cd->
columnName +
" " +
3715 "' column types do not match.");
3729 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3731 "' and target '" + target_cd->
columnName +
" " +
3733 "' decimal columns scales do not match.");
3739 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3741 "' and target '" + target_cd->
columnName +
" " +
3743 "' column types do not match.");
3747 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3749 "' and target '" + target_cd->
columnName +
" " +
3751 "' columns string encodings do not match.");
3758 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3760 "' and target '" + target_cd->
columnName +
" " +
3762 "' timestamp column precisions do not match.");
3771 throw std::runtime_error(
"Source '" + source_cd->
columnName +
" " +
3773 "' and target '" + target_cd->
columnName +
" " +
3775 "' column encoding sizes do not match.");
3780 if (!populate_table) {
3784 int64_t total_row_count = 0;
3785 int64_t total_source_query_time_ms = 0;
3786 int64_t total_target_value_translate_time_ms = 0;
3787 int64_t total_data_load_time_ms = 0;
3790 auto target_column_descriptors = get_target_column_descriptors(td);
3791 auto outer_frag_count =
3794 size_t outer_frag_end = outer_frag_count == 0 ? 1 : outer_frag_count;
3795 auto query_session = session ? session->get_session_id() :
"";
3797 std::string work_type_str = for_CTAS ?
"CTAS" :
"ITAS";
3799 for (
size_t outer_frag_idx = 0; outer_frag_idx < outer_frag_end; outer_frag_idx++) {
3800 std::vector<size_t> allowed_outer_fragment_indices;
3802 if (outer_frag_count) {
3803 allowed_outer_fragment_indices.push_back(outer_frag_idx);
3807 std::vector<AggregatedResult> query_results =
3810 allowed_outer_fragment_indices,
3812 total_source_query_time_ms +=
timer_stop(query_clock_begin);
3815 auto query_str =
"INSERT_DATA for " + work_type_str;
3820 executor->enrollQuerySession(query_session,
3824 QuerySessionStatus::QueryStatus::RUNNING_IMPORTER);
3831 executor->clearQuerySessionStatus(query_session,
start_time);
3835 for (
auto&
res : query_results) {
3837 throw std::runtime_error(
3838 "Query execution has been interrupted while performing " + work_type_str);
3840 auto& result_rows =
res.rs;
3842 const auto num_rows = result_rows->rowCount();
3844 if (0 == num_rows) {
3848 total_row_count += num_rows;
3853 const size_t rows_per_block =
3854 std::max(std::min(num_rows / leaf_count,
size_t(64 * 1024)),
size_t(1));
3856 std::vector<std::unique_ptr<TargetValueConverter>> value_converters;
3860 const int num_worker_threads = std::thread::hardware_concurrency();
3862 std::vector<size_t> thread_start_idx(num_worker_threads),
3863 thread_end_idx(num_worker_threads);
3864 bool can_go_parallel = !result_rows->isTruncated() && rows_per_block > 20000;
3866 std::atomic<size_t> crt_row_idx{0};
3868 auto do_work = [&result_rows, &value_converters, &crt_row_idx](
3870 const size_t block_end,
3871 const size_t num_cols,
3873 bool& stop_convert) {
3874 const auto result_row = result_rows->getRowAtNoTranslations(idx);
3875 if (!result_row.empty()) {
3876 size_t target_row = crt_row_idx.fetch_add(1);
3877 if (target_row >= block_end) {
3878 stop_convert =
true;
3881 for (
unsigned int col = 0; col < num_cols; col++) {
3882 const auto& mapd_variant = result_row[col];
3883 value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
3888 auto convert_function = [&thread_start_idx,
3894 &do_work](
const int thread_id,
const size_t block_end) {
3895 const int num_cols = value_converters.size();
3896 const size_t start = thread_start_idx[
thread_id];
3897 const size_t end = thread_end_idx[
thread_id];
3899 bool stop_convert =
false;
3901 size_t local_idx = 0;
3902 for (idx = start; idx < end; ++idx, ++local_idx) {
3903 if (
UNLIKELY((local_idx & 0xFFFF) == 0 &&
3905 throw std::runtime_error(
3906 "Query execution has been interrupted while performing " +
3909 do_work(idx, block_end, num_cols, thread_id, stop_convert);
3915 for (idx = start; idx < end; ++idx) {
3916 do_work(idx, block_end, num_cols, thread_id, stop_convert);
3925 auto single_threaded_value_converter =
3926 [&crt_row_idx, &value_converters, &result_rows](
const size_t idx,
3927 const size_t block_end,
3928 const size_t num_cols,
3929 bool& stop_convert) {
3930 size_t target_row = crt_row_idx.fetch_add(1);
3931 if (target_row >= block_end) {
3932 stop_convert =
true;
3935 const auto result_row = result_rows->getNextRow(
false,
false);
3936 CHECK(!result_row.empty());
3937 for (
unsigned int col = 0; col < num_cols; col++) {
3938 const auto& mapd_variant = result_row[col];
3939 value_converters[col]->convertToColumnarFormat(target_row, &mapd_variant);
3943 auto single_threaded_convert_function = [&value_converters,
3949 &single_threaded_value_converter](
3951 const size_t block_end) {
3952 const int num_cols = value_converters.size();
3953 const size_t start = thread_start_idx[
thread_id];
3954 const size_t end = thread_end_idx[
thread_id];
3956 bool stop_convert =
false;
3958 size_t local_idx = 0;
3959 for (idx = start; idx < end; ++idx, ++local_idx) {
3960 if (
UNLIKELY((local_idx & 0xFFFF) == 0 &&
3962 throw std::runtime_error(
3963 "Query execution has been interrupted while performing " +
3966 single_threaded_value_converter(idx, block_end, num_cols, stop_convert);
3972 for (idx = start; idx < end; ++idx) {
3973 single_threaded_value_converter(idx, end, num_cols, stop_convert);
3982 if (can_go_parallel) {
3983 const size_t entry_count = result_rows->entryCount();
3987 stride = (entry_count + num_worker_threads - 1) / num_worker_threads;
3988 i < num_worker_threads && start_entry < entry_count;
3989 ++i, start_entry += stride) {
3990 const auto end_entry = std::min(start_entry + stride, entry_count);
3991 thread_start_idx[i] = start_entry;
3992 thread_end_idx[i] = end_entry;
3995 thread_start_idx[0] = 0;
3996 thread_end_idx[0] = result_rows->entryCount();
4001 for (
size_t block_start = 0; block_start < num_rows;
4002 block_start += rows_per_block) {
4003 const auto num_rows_this_itr = block_start + rows_per_block < num_rows
4005 : num_rows - block_start;
4007 value_converters.clear();
4009 for (
const auto targetDescriptor : target_column_descriptors) {
4010 auto sourceDataMetaInfo =
res.targets_meta[colNum++];
4016 targetDescriptor->columnType,
4017 !targetDescriptor->columnType.get_notnull(),
4018 result_rows->getRowSetMemOwner()->getLiteralStringDictProxy(),
4020 sourceDataMetaInfo.get_type_info().is_dict_encoded_string()
4021 ? executor->getStringDictionaryProxy(
4022 sourceDataMetaInfo.get_type_info().get_comp_param(),
4023 result_rows->getRowSetMemOwner(),
4026 IS_GEO_POLY(targetDescriptor->columnType.get_type()) &&
4028 ? &render_group_analyzer_map
4030 auto converter = factory.
create(param);
4031 value_converters.push_back(std::move(converter));
4035 if (can_go_parallel) {
4036 std::vector<std::future<void>> worker_threads;
4037 for (
int i = 0; i < num_worker_threads; ++i) {
4038 worker_threads.push_back(
4042 for (
auto& child : worker_threads) {
4045 for (
auto& child : worker_threads) {
4050 single_threaded_convert_function(0, num_rows_this_itr);
4054 auto finalizer_func =
4055 [](std::unique_ptr<TargetValueConverter>::pointer targetValueConverter) {
4056 targetValueConverter->finalizeDataBlocksForInsertData();
4059 std::vector<std::future<void>> worker_threads;
4060 for (
auto& converterPtr : value_converters) {
4061 worker_threads.push_back(
4065 for (
auto& child : worker_threads) {
4068 for (
auto& child : worker_threads) {
4073 insert_data.
databaseId = catalog.getCurrentDB().dbId;
4076 insert_data.
numRows = num_rows_this_itr;
4078 for (
int col_idx = 0; col_idx < target_column_descriptors.size(); col_idx++) {
4081 throw std::runtime_error(
4082 "Query execution has been interrupted while performing " +
4085 value_converters[col_idx]->addDataBlocksToInsertData(insert_data);
4087 total_target_value_translate_time_ms +=
timer_stop(translate_clock_begin);
4090 auto data_memory_holder =
4092 insertDataLoader.insertData(*session, insert_data);
4093 total_data_load_time_ms +=
timer_stop(data_load_clock_begin);
4100 }
catch (std::exception& e) {
4101 LOG(
ERROR) <<
"An error occurred during ITAS rollback attempt. Table id: "
4102 << td->
tableId <<
", Error: " << e.what();
4107 int64_t total_time_ms = total_source_query_time_ms +
4108 total_target_value_translate_time_ms + total_data_load_time_ms;
4110 VLOG(1) <<
"CTAS/ITAS " << total_row_count <<
" rows loaded in " << total_time_ms
4111 <<
"ms (outer_frag_count=" << outer_frag_count
4112 <<
", query_time=" << total_source_query_time_ms
4113 <<
"ms, translation_time=" << total_target_value_translate_time_ms
4114 <<
"ms, data_load_time=" << total_data_load_time_ms
4117 if (!is_temporary) {
void validate_non_foreign_table_write(const TableDescriptor *table_descriptor)
HOST DEVICE SQLTypes get_subtype() const
HOST DEVICE int get_size() const
bool is_timestamp() const
HOST DEVICE int get_scale() const
static const AccessPrivileges INSERT_INTO_TABLE
std::vector< std::unique_ptr< TypedImportBuffer > > fill_missing_columns(const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data)
TypeR::rep timer_stop(Type clock_begin)
bool g_enable_non_kernel_time_query_interrupt
HOST DEVICE SQLTypes get_type() const
bool g_enable_string_functions
int tableId
identifies the database into which the data is being inserted
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
size_t numRows
a vector of column ids for the row(s) being inserted
future< Result > async(Fn &&fn, Args &&...args)
std::unique_ptr< QueryConnector > leafs_connector_
bool g_enable_assign_render_groups
QueryState & getQueryState()
specifies the content in-memory of a row in the column metadata table
bool check_session_interrupted(const QuerySessionId &query_session, Executor *executor)
HOST DEVICE EncodingType get_compression() const
bool table_is_temporary(const TableDescriptor *const td)
HOST DEVICE int get_dimension() const
std::string select_query_
std::string get_type_name() const
std::vector< std::unique_ptr< std::string > > column_list_
The data to be inserted using the fragment manager.
std::map< int, import_export::RenderGroupAnalyzer > RenderGroupAnalyzerMap
SQLTypeInfo get_elem_type() const
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
const std::string getQuerySubmittedTime() const
std::unique_ptr< TargetValueConverter > create(ConverterCreateParameter param)
static const ExecutorId UNITARY_EXECUTOR_ID