33 const size_t groups_buffer_entry_count = query_mem_desc.
getEntryCount();
35 const size_t num_count_distinct_descs =
37 for (
size_t i = 0; i < num_count_distinct_descs; i++) {
44 int64_t total_bytes{0};
48 total_bytes =
static_cast<int64_t
>(total_bytes_per_group * groups_buffer_entry_count);
60 const size_t numBytes,
62 const size_t thread_idx,
64 const bool reuse_existing_buffer_for_thread) {
65 if (render_allocator_map) {
69 const auto gpu_idx = 0;
71 return std::make_pair(
72 reinterpret_cast<int64_t*>(render_allocator_ptr->alloc(numBytes)),
false);
73 }
else if (reuse_existing_buffer_for_thread) {
76 return std::make_pair(
77 reinterpret_cast<int64_t*>(mem_owner->
allocate(numBytes, thread_idx)),
false);
81 if (frag_offsets.size() < 2) {
84 const auto frag_size = frag_offsets[1] - frag_offsets[0];
85 for (
size_t i = 2; i < frag_offsets.size(); ++i) {
86 const auto curr_size = frag_offsets[i] - frag_offsets[i - 1];
87 if (curr_size != frag_size) {
91 return !frag_size ? std::numeric_limits<int64_t>::max()
92 :
static_cast<int64_t
>(frag_size);
96 const std::vector<std::vector<uint64_t>>& frag_offsets) {
97 if (frag_offsets.empty()) {
100 std::vector<int64_t> frag_sizes;
101 for (
size_t tab_idx = 0; tab_idx < frag_offsets[0].size(); ++tab_idx) {
102 std::vector<uint64_t> tab_offs;
103 for (
auto& offsets : frag_offsets) {
104 tab_offs.push_back(offsets[tab_idx]);
112 const std::vector<Analyzer::Expr*>& target_exprs,
113 const std::vector<int64_t>& table_frag_sizes) {
114 std::vector<int64_t> col_frag_sizes;
115 for (
auto expr : target_exprs) {
116 if (
const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
117 if (col_var->get_rte_idx() < 0) {
118 CHECK_EQ(-1, col_var->get_rte_idx());
119 col_frag_sizes.push_back(int64_t(-1));
121 col_frag_sizes.push_back(table_frag_sizes[col_var->get_rte_idx()]);
124 col_frag_sizes.push_back(int64_t(-1));
127 return col_frag_sizes;
131 const std::vector<Analyzer::Expr*>& target_exprs,
132 const std::vector<std::vector<uint64_t>>& table_frag_offsets) {
133 std::vector<std::vector<int64_t>> col_frag_offsets;
134 for (
auto& table_offsets : table_frag_offsets) {
135 std::vector<int64_t> col_offsets;
136 for (
auto expr : target_exprs) {
137 if (
const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
138 if (col_var->get_rte_idx() < 0) {
139 CHECK_EQ(-1, col_var->get_rte_idx());
140 col_offsets.push_back(int64_t(-1));
142 CHECK_LT(static_cast<size_t>(col_var->get_rte_idx()), table_offsets.size());
143 col_offsets.push_back(
144 static_cast<int64_t>(table_offsets[col_var->get_rte_idx()]));
147 col_offsets.push_back(int64_t(-1));
150 col_frag_offsets.push_back(col_offsets);
152 return col_frag_offsets;
159 auto match_table_key = [=](
auto& desc) {
160 return outer_table_key == desc.getTableKey();
163 auto itr = std::find_if(input_descs.begin(), input_descs.end(), match_table_key);
164 return itr == input_descs.end() ? 0 : itr->getNestLevel();
169 const size_t agg_col_count{query_mem_desc.
getSlotCount()};
171 for (
size_t target_idx = 0; target_idx < ra_exe_unit.
target_exprs.size();
173 const auto target_expr = ra_exe_unit.
target_exprs[target_idx];
176 CHECK(agg_info.is_agg &&
179 CHECK(!agg_info.sql_type.is_varlen());
181 CHECK_LT(static_cast<size_t>(agg_col_idx), agg_col_count);
184 const auto& count_distinct_desc =
202 if (
auto const* agg_expr = dynamic_cast<Analyzer::AggExpr const*>(expr)) {
203 if (agg_expr->get_aggtype() ==
kMODE) {
210 return agg_op_metadata;
222 const bool output_columnar,
225 const int64_t num_rows,
226 const std::vector<std::vector<const int8_t*>>& col_buffers,
227 const std::vector<std::vector<uint64_t>>& frag_offsets,
230 std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
232 const size_t thread_idx,
233 const Executor* executor)
234 : num_rows_(num_rows)
235 , row_set_mem_owner_(row_set_mem_owner)
236 , init_agg_vals_(executor->plan_state_->init_agg_vals_)
237 ,
num_buffers_(computeNumberOfBuffers(query_mem_desc, device_type, executor))
246 CHECK(!sort_on_gpu || output_columnar);
247 executor->logSystemCPUMemoryStatus(
"Before Query Memory Initialization", thread_idx);
250 if (consistent_frag_sizes.empty()) {
267 if (render_allocator_map || !query_mem_desc.
isGroupBy()) {
302 ? executor->blockSize() * executor->gridSize()
305 size_t group_buffer_size{0};
310 group_buffer_size = num_rows * query_mem_desc.
getRowSize();
319 CHECK_GE(group_buffer_size,
size_t(0));
322 int64_t* group_by_buffer_template{
nullptr};
323 if (!query_mem_desc.
lazyInitGroups(device_type) && group_buffers_count > 1) {
324 group_by_buffer_template =
reinterpret_cast<int64_t*
>(
342 ? executor->blockSize()
348 const auto actual_group_buffer_size =
349 group_buffer_size + index_buffer_qw *
sizeof(int64_t);
350 CHECK_GE(actual_group_buffer_size, group_buffer_size);
354 CHECK(varlen_buffer_elem_size_opt);
355 auto const varlen_buffer_sz =
356 query_mem_desc.
getEntryCount() * varlen_buffer_elem_size_opt.value();
357 auto varlen_output_buffer =
370 CHECK_EQ(group_buffers_count,
size_t(1));
377 for (
size_t i = 0; i < group_buffers_count; i += step) {
380 render_allocator_map,
385 auto group_by_buffer = group_by_info.first;
386 const bool was_cached = group_by_info.second;
389 if (group_by_buffer_template) {
390 memcpy(group_by_buffer + index_buffer_qw,
391 group_by_buffer_template,
409 const bool use_target_exprs_union =
414 const auto column_frag_sizes =
418 result_sets_.resize(old_size + std::max(
size_t(1), step));
421 executor->getColLazyFetchInfo(target_exprs),
430 executor->blockSize(),
431 executor->gridSize());
432 result_sets_[old_size]->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
433 executor->plan_state_->init_agg_vals_,
444 const int64_t num_rows,
445 const std::vector<std::vector<const int8_t*>>& col_buffers,
446 const std::vector<std::vector<uint64_t>>& frag_offsets,
447 std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
449 const Executor* executor)
450 : num_rows_(num_rows)
451 , row_set_mem_owner_(row_set_mem_owner)
464 if (consistent_frag_sizes.empty()) {
469 const size_t num_columns =
471 size_t total_group_by_buffer_size{0};
472 for (
size_t i = 0; i < num_columns; ++i) {
473 auto ti = exe_unit.target_exprs[i]->get_type_info();
474 if (ti.usesFlatBuffer()) {
482 static_cast<int64_t>(total_group_by_buffer_size + flatbuffer_size));
491 static_cast<int64_t>(group_buffer_size + total_group_by_buffer_size));
498 #ifdef __SANITIZE_ADDRESS__
500 #define MAX_BUFFER_SIZE 0x10000000000ll
504 #define MAX_BUFFER_SIZE 0x100000000000ll
515 row_set_mem_owner.get(),
518 group_by_buffers_.push_back(group_by_buffer);
520 const auto column_frag_offsets =
522 const auto column_frag_sizes =
524 result_sets_.emplace_back(
535 executor->blockSize(),
536 executor->gridSize()));
537 result_sets_.back()->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
547 const bool output_columnar,
548 const Executor* executor) {
549 if (output_columnar) {
552 auto rows_ptr = buffer;
555 ? executor->blockSize() * executor->gridSize()
558 query_mem_desc.
interleavedBins(device_type) ? executor->warpSize() : 1;
560 const auto node_count_size = thread_count *
sizeof(int64_t);
561 memset(rows_ptr, 0, node_count_size);
565 memset(rows_ptr + thread_count, -1, rows_offset - node_count_size);
566 rows_ptr += rows_offset /
sizeof(int64_t);
567 actual_entry_count =
n * thread_count;
582 int64_t* groups_buffer,
583 const std::vector<int64_t>& init_vals,
585 const int32_t groups_buffer_entry_count,
586 const size_t warp_size,
587 const Executor* executor,
590 const size_t row_size{query_mem_desc.
getRowSize()};
593 auto buffer_ptr =
reinterpret_cast<int8_t*
>(groups_buffer);
594 const auto query_mem_desc_fixedup =
602 std::vector<int8_t> sample_row(row_size - col_base_off);
603 auto const num_available_cpu_threads =
606 tbb::task_arena initialization_arena(num_available_cpu_threads);
609 query_mem_desc_fixedup, sample_row.data(), init_vals, agg_op_metadata);
612 CHECK(warp_size >= 1);
613 CHECK(key_count == 1 || warp_size == 1);
614 initialization_arena.execute([&] {
616 tbb::blocked_range<size_t>(0, groups_buffer_entry_count * warp_size),
617 [&](
const tbb::blocked_range<size_t>& r) {
618 auto cur_row_buf = buffer_ptr + (row_size * r.begin());
619 for (
size_t i = r.begin(); i != r.end(); ++i, cur_row_buf += row_size) {
620 memcpy(cur_row_buf + col_base_off, sample_row.data(), sample_row.size());
626 initialization_arena.execute([&] {
628 tbb::blocked_range<size_t>(0, groups_buffer_entry_count),
629 [&](
const tbb::blocked_range<size_t>& r) {
630 auto cur_row_buf = buffer_ptr + (row_size * r.begin());
631 for (
size_t i = r.begin(); i != r.end(); ++i, cur_row_buf += row_size) {
632 memcpy(cur_row_buf + col_base_off, sample_row.data(), sample_row.size());
640 CHECK(warp_size >= 1);
641 CHECK(key_count == 1 || warp_size == 1);
642 for (
size_t warp_idx = 0; warp_idx < warp_size; ++warp_idx) {
643 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
644 ++bin, buffer_ptr += row_size) {
646 &buffer_ptr[col_base_off],
654 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
655 ++bin, buffer_ptr += row_size) {
659 query_mem_desc_fixedup, &buffer_ptr[col_base_off], init_vals, agg_op_metadata);
666 template <
typename T>
668 static_assert(
sizeof(
T) <=
sizeof(int64_t),
"Unsupported template type");
669 for (uint32_t i = 0; i < entry_count; ++i) {
670 buffer_ptr[i] = init_val;
672 return reinterpret_cast<int8_t*
>(buffer_ptr + entry_count);
679 int64_t* groups_buffer,
680 const std::vector<int64_t>& init_vals,
681 const Executor* executor,
683 CHECK(groups_buffer);
685 for (
const auto target_expr : ra_exe_unit.
target_exprs) {
689 const int32_t agg_col_count = query_mem_desc.
getSlotCount();
690 auto buffer_ptr =
reinterpret_cast<int8_t*
>(groups_buffer);
692 const auto groups_buffer_entry_count = query_mem_desc.
getEntryCount();
695 for (
size_t i = 0; i < key_count; ++i) {
696 buffer_ptr = initColumnarBuffer<int64_t>(
reinterpret_cast<int64_t*
>(buffer_ptr),
698 groups_buffer_entry_count);
704 int32_t init_val_idx = 0;
705 for (int32_t i = 0; i < agg_col_count; ++i) {
707 CHECK_LT(static_cast<size_t>(init_val_idx), init_vals.size());
710 buffer_ptr = initColumnarBuffer<int8_t>(
711 buffer_ptr, init_vals[init_val_idx++], groups_buffer_entry_count);
715 initColumnarBuffer<int16_t>(
reinterpret_cast<int16_t*
>(buffer_ptr),
716 init_vals[init_val_idx++],
717 groups_buffer_entry_count);
721 initColumnarBuffer<int32_t>(
reinterpret_cast<int32_t*
>(buffer_ptr),
722 init_vals[init_val_idx++],
723 groups_buffer_entry_count);
727 initColumnarBuffer<int64_t>(
reinterpret_cast<int64_t*
>(buffer_ptr),
728 init_vals[init_val_idx++],
729 groups_buffer_entry_count);
746 const std::vector<int64_t>& init_vals,
748 int8_t* col_ptr = row_ptr;
749 size_t init_vec_idx = 0;
750 for (
size_t col_idx = 0; col_idx < query_mem_desc.
getSlotCount();
773 }
else if (agg_op_metadata.
has_mode &&
781 if (init_val == 0 && col_slot_width > 0) {
782 CHECK_LT(init_vec_idx, init_vals.size());
783 init_val = init_vals[init_vec_idx++];
785 switch (col_slot_width) {
787 *col_ptr =
static_cast<int8_t
>(init_val);
790 *
reinterpret_cast<int16_t*
>(col_ptr) = (int16_t)init_val;
793 *
reinterpret_cast<int32_t*
>(col_ptr) = (int32_t)init_val;
796 *
reinterpret_cast<int64_t*
>(col_ptr) = init_val;
813 size_t total_bytes_per_entry{0};
814 const size_t num_count_distinct_descs =
816 for (
size_t i = 0; i < num_count_distinct_descs; i++) {
822 total_bytes_per_entry += count_distinct_desc.bitmapPaddedSizeBytes();
839 const size_t agg_col_count{query_mem_desc.
getSlotCount()};
840 std::vector<int64_t> agg_bitmap_size(agg_col_count);
841 for (
size_t target_idx = 0; target_idx < ra_exe_unit.
target_exprs.size();
843 const auto target_expr = ra_exe_unit.
target_exprs[target_idx];
847 const auto& count_distinct_desc =
851 agg_bitmap_size[agg_col_idx] = bitmap_byte_sz;
854 agg_bitmap_size[agg_col_idx] = -1;
858 return agg_bitmap_size;
864 for (
size_t target_idx = 0; target_idx < ra_exe_unit.
target_exprs.size();
866 const auto target_expr = ra_exe_unit.
target_exprs[target_idx];
870 const auto& count_distinct_desc =
889 ptr, bitmap_byte_sz,
false);
890 return reinterpret_cast<int64_t
>(ptr);
892 return reinterpret_cast<int64_t
>(
899 return reinterpret_cast<int64_t
>(count_distinct_set);
905 std::vector<Analyzer::Expr*>
const& target_exprs,
908 for (
size_t target_idx = 0; target_idx < target_exprs.size(); ++target_idx) {
909 auto const target_expr = target_exprs[target_idx];
910 if (
auto const* agg_expr = dynamic_cast<Analyzer::AggExpr const*>(target_expr)) {
911 if (agg_expr->get_aggtype() == agg_type) {
912 lambda(agg_expr, target_idx);
923 size_t const slot_count = query_mem_desc.
getSlotCount();
930 size_t const agg_col_idx =
933 mode_index_set.emplace(agg_col_idx);
935 return mode_index_set;
941 size_t const slot_count = query_mem_desc.
getSlotCount();
947 size_t const agg_col_idx =
951 init_agg_vals_[agg_col_idx] =
reinterpret_cast<int64_t
>(agg_mode);
955 std::vector<QueryMemoryInitializer::QuantileParam>
959 size_t const slot_count = query_mem_desc.
getSlotCount();
961 std::vector<QuantileParam> quantile_params(slot_count);
966 size_t const agg_col_idx =
969 CHECK_EQ(static_cast<int8_t>(
sizeof(int64_t)),
974 quantile_params[agg_col_idx] = q_expr->get_constval().doubleval;
976 return quantile_params;
982 size_t const slot_count = query_mem_desc.
getSlotCount();
988 size_t const agg_col_idx =
991 CHECK_EQ(static_cast<int8_t>(
sizeof(int64_t)),
996 auto const q = q_expr->get_constval().doubleval;
1005 const int8_t* init_agg_vals_dev_ptr,
1007 const int device_id,
1008 const unsigned block_size_x,
1009 const unsigned grid_size_x) {
1012 const auto thread_count = block_size_x * grid_size_x;
1013 const auto total_buff_size =
1017 std::vector<int8_t*> dev_buffers(thread_count);
1019 for (
size_t i = 0; i < thread_count; ++i) {
1020 dev_buffers[i] = dev_buffer;
1025 dev_ptr, dev_buffers.data(), thread_count *
sizeof(int8_t*));
1030 thread_count *
sizeof(int64_t));
1033 reinterpret_cast<int8_t*>(dev_buffer + thread_count *
sizeof(int64_t)),
1035 thread_count * n *
sizeof(int64_t));
1038 reinterpret_cast<int64_t*>(
1040 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
1044 query_mem_desc.
getRowSize() /
sizeof(int64_t),
1050 return {dev_ptr, dev_buffer};
1060 const int8_t* init_agg_vals_dev_ptr,
1061 const int device_id,
1063 const unsigned block_size_x,
1064 const unsigned grid_size_x,
1065 const int8_t warp_size,
1066 const bool can_sort_on_gpu,
1067 const bool output_columnar,
1071 if (render_allocator) {
1075 CHECK(!output_columnar);
1078 query_mem_desc, init_agg_vals_dev_ptr,
n, device_id, block_size_x, grid_size_x);
1081 auto dev_group_by_buffers =
1096 CHECK(dev_group_by_buffers.varlen_output_buffer);
1098 reinterpret_cast<CUdeviceptr>(dev_group_by_buffers.varlen_output_buffer);
1100 const size_t varlen_output_buf_bytes =
1109 if (render_allocator) {
1113 CHECK(!render_allocator);
1118 auto group_by_dev_buffer = dev_group_by_buffers.data;
1119 const size_t col_count = query_mem_desc.
getSlotCount();
1120 int8_t* col_widths_dev_ptr{
nullptr};
1121 if (output_columnar) {
1122 std::vector<int8_t> compact_col_widths(col_count);
1123 for (
size_t idx = 0; idx < col_count; ++idx) {
1128 col_widths_dev_ptr, compact_col_widths.data(), col_count *
sizeof(int8_t));
1130 const int8_t warp_count =
1132 const auto num_group_by_buffers =
1134 for (
size_t i = 0; i < num_group_by_buffers; i += step) {
1135 if (output_columnar) {
1137 reinterpret_cast<int64_t*>(group_by_dev_buffer),
1138 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
1139 dev_group_by_buffers.entry_count,
1150 reinterpret_cast<int64_t*>(group_by_dev_buffer),
1151 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
1152 dev_group_by_buffers.entry_count,
1155 query_mem_desc.
getRowSize() /
sizeof(int64_t),
1161 group_by_dev_buffer += groups_buffer_size;
1164 return dev_group_by_buffers;
1173 const int device_id,
1174 const unsigned block_size_x,
1175 const unsigned grid_size_x,
1176 const bool zero_initialize_buffers) {
1179 size_t total_group_by_buffer_size{0};
1182 std::vector<size_t> col_byte_offsets;
1183 col_byte_offsets.reserve(num_columns);
1185 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1187 size_t group_buffer_size =
num_rows_ * col_width;
1188 col_byte_offsets.emplace_back(total_group_by_buffer_size);
1189 total_group_by_buffer_size =
1193 int8_t* dev_buffers_allocation{
nullptr};
1195 CHECK(dev_buffers_allocation);
1196 if (zero_initialize_buffers) {
1200 auto dev_buffers_mem = dev_buffers_allocation;
1201 std::vector<int8_t*> dev_buffers(num_columns);
1202 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1203 dev_buffers[col_idx] = dev_buffers_allocation + col_byte_offsets[col_idx];
1207 dev_ptrs, dev_buffers.data(), num_columns *
sizeof(
CUdeviceptr));
1209 return {dev_ptrs, dev_buffers_mem, (size_t)
num_rows_};
1215 const size_t entry_count,
1217 const int device_id,
1218 const unsigned block_size_x,
1219 const unsigned grid_size_x) {
1222 int8_t* dev_buffer = gpu_group_by_buffers.
data;
1225 const size_t original_entry_count = gpu_group_by_buffers.
entry_count;
1226 CHECK_LE(entry_count, original_entry_count);
1227 size_t output_device_col_offset{0};
1228 size_t output_host_col_offset{0};
1232 auto allocator = std::make_unique<CudaAllocator>(
1235 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1237 const size_t output_device_col_size = original_entry_count * col_width;
1238 const size_t output_host_col_size = entry_count * col_width;
1239 allocator->copyFromDevice(host_buffer + output_host_col_offset,
1240 dev_buffer + output_device_col_offset,
1241 output_host_col_size);
1242 output_device_col_offset =
1243 align_to_int64(output_device_col_offset + output_device_col_size);
1244 output_host_col_offset =
1255 : executor->blockSize() *
1264 int8_t* projection_buffer,
1265 const size_t projection_count) {
1268 constexpr
size_t row_index_width =
sizeof(int64_t);
1269 size_t buffer_offset1{projection_count * row_index_width};
1271 for (
size_t i = 0; i < query_mem_desc.
getSlotCount(); i++) {
1273 auto column_proj_size =
1276 if (buffer_offset1 + column_proj_size >= buffer_offset2) {
1278 std::memmove(projection_buffer + buffer_offset1,
1279 projection_buffer + buffer_offset2,
1282 std::memcpy(projection_buffer + buffer_offset1,
1283 projection_buffer + buffer_offset2,
1295 const size_t projection_count) {
1296 const auto num_allocated_rows =
1298 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1304 num_allocated_rows);
1308 result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1315 const size_t projection_count,
1316 const int device_id) {
1318 const auto num_allocated_rows =
1322 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1325 gpu_group_by_buffers,
1333 result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1339 const size_t entry_count,
1342 const unsigned block_size_x,
1343 const unsigned grid_size_x,
1344 const int device_id,
1345 const bool prepend_index_buffer)
const {
1346 const auto thread_count = block_size_x * grid_size_x;
1348 size_t total_buff_size{0};
1361 gpu_group_by_buffers.
data,
1366 prepend_index_buffer,
1373 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1391 const unsigned total_thread_count,
1392 const int device_id) {
1395 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1397 const auto rows_copy = pick_top_n_rows_from_dev_heaps(
1399 reinterpret_cast<int64_t*>(gpu_group_by_buffers.
data),
GpuGroupByBuffers setupTableFunctionGpuBuffers(const QueryMemoryDescriptor &query_mem_desc, const int device_id, const unsigned block_size_x, const unsigned grid_size_x, const bool zero_initialize_buffers)
ModeIndexSet initializeModeIndexSet(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
std::vector< Analyzer::Expr * > target_exprs
size_t getSlotCount() const
size_t getBufferSizeBytes(const RelAlgExecutionUnit &ra_exe_unit, const unsigned thread_count, const ExecutorDeviceType device_type) const
GpuGroupByBuffers create_dev_group_by_buffers(DeviceAllocator *device_allocator, const std::vector< int64_t * > &group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const ExecutorDispatchMode dispatch_mode, const int64_t num_input_rows, const bool prepend_index_buffer, const bool always_init_group_by_on_host, const bool use_bump_allocator, const bool has_varlen_output, Allocator *insitu_allocator)
RenderAllocator * getRenderAllocator(size_t device_id)
robin_hood::unordered_set< int64_t > CountDistinctSet
bool countDistinctDescriptorsLogicallyEmpty() const
size_t getEntryCount() const
bool useCudaBuffers() const
CUdeviceptr count_distinct_bitmap_device_mem_ptr_
GpuGroupByBuffers prepareTopNHeapsDevBuffer(const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const size_t n, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
QueryMemoryInitializer::TargetAggOpsMetadata collect_target_expr_metadata(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
GpuGroupByBuffers createAndInitializeGroupByBufferGpu(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const int device_id, const ExecutorDispatchMode dispatch_mode, const unsigned block_size_x, const unsigned grid_size_x, const int8_t warp_size, const bool can_sort_on_gpu, const bool output_columnar, RenderAllocator *render_allocator)
boost::multiprecision::number< boost::multiprecision::cpp_int_backend< 64, 64, boost::multiprecision::signed_magnitude, boost::multiprecision::checked, void >> checked_int64_t
void compact_projection_buffer_for_cpu_columnar(const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count)
DeviceAllocator * device_allocator_
size_t getAvailableCpuThreads() const
count_distinct_bitmap_host_mem_ptr_(nullptr)
void sort_on_gpu(int64_t *val_buff, int32_t *idx_buff, const uint64_t entry_count, const bool desc, const uint32_t chosen_bytes, ThrustAllocator &alloc, const int device_id)
const std::optional< bool > union_all
Streaming Top N algorithm.
size_t get_rows_offset_of_heaps(const size_t n, const size_t thread_count)
void allocateCountDistinctBuffers(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
bool hasVarlenOutput() const
QueryMemoryInitializer(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int device_id, const ExecutorDeviceType device_type, const ExecutorDispatchMode dispatch_mode, const bool output_columnar, const bool sort_on_gpu, const shared::TableKey &outer_table_key, const int64_t num_rows, const std::vector< std::vector< const int8_t * >> &col_buffers, const std::vector< std::vector< uint64_t >> &frag_offsets, RenderAllocatorMap *render_allocator_map, RenderInfo *render_info, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, DeviceAllocator *gpu_allocator, const size_t thread_idx, const Executor *executor)
unsigned long long CUdeviceptr
int8_t * allocate(const size_t num_bytes, const size_t thread_idx=0) override
std::vector< InputDescriptor > input_descs
bool hasKeylessHash() const
void init_columnar_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t agg_col_count, const int8_t *col_sizes, const bool need_padding, const bool keyless, const int8_t key_size, const size_t block_size_x, const size_t grid_size_x)
count_distinct_bitmap_device_mem_ptr_(0)
void check_total_bitmap_memory(const QueryMemoryDescriptor &query_mem_desc)
virtual int8_t * alloc(const size_t num_bytes)=0
size_t getEffectiveKeyWidth() const
void eachAggregateTargetIdxOfType(std::vector< Analyzer::Expr * > const &target_exprs, SQLAgg const agg_type, std::function< void(Analyzer::AggExpr const *, size_t)> lambda)
int8_t * initColumnarBuffer(T *buffer_ptr, const T init_val, const uint32_t entry_count)
count_distinct_bitmap_mem_size_(0)
TargetInfo get_target_info(const Analyzer::Expr *target_expr, const bool bigint_count)
size_t computeNumberOfBuffers(const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const Executor *executor) const
std::vector< QuantileParam > initializeQuantileParams(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
size_t getRowSize() const
std::pair< int64_t *, bool > allocateCachedGroupByBuffer(const size_t num_bytes, const size_t thread_idx)
void check_count_distinct_expr_metadata(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
varlen_output_buffer_host_ptr_(nullptr)
void init_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad, const bool keyless, const int8_t warp_size, const size_t block_size_x, const size_t grid_size_x)
const SlotSize & getSlotInfo(const size_t slot_idx) const
std::vector< Analyzer::Expr * > target_exprs_union
size_t count_distinct_bitmap_mem_size_
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
void compactProjectionBuffersGpu(const QueryMemoryDescriptor &query_mem_desc, Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const size_t projection_count, const int device_id)
bool blocksShareMemory() const
std::pair< int64_t *, bool > alloc_group_by_buffer(const size_t numBytes, RenderAllocatorMap *render_allocator_map, const size_t thread_idx, RowSetMemoryOwner *mem_owner, const bool reuse_existing_buffer_for_thread)
virtual void copyToDevice(void *device_dst, const void *host_src, const size_t num_bytes) const =0
std::vector< int64_t > calculateCountDistinctBufferSize(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit) const
int8_t * count_distinct_bitmap_host_mem_ptr_
std::vector< int64_t > init_agg_vals_
bool useStreamingTopN() const
size_t getGroupbyColCount() const
void applyStreamingTopNOffsetCpu(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
void fill_empty_key(void *key_ptr, const size_t key_count, const size_t key_width)
virtual void zeroDeviceMem(int8_t *device_ptr, const size_t num_bytes) const =0
bool lazyInitGroups(const ExecutorDeviceType) const
bool threadsCanReuseGroupByBuffers() const
std::optional< size_t > limit
count_distinct_bitmap_host_crt_ptr_(nullptr)
int64_t g_bitmap_memory_limit
size_t g_max_memory_allocation_size
size_t getAllocatedSize() const
bool is_distinct_target(const TargetInfo &target_info)
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
const std::shared_ptr< Analyzer::Estimator > estimator
void initColumnarGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const Executor *executor, const RelAlgExecutionUnit &ra_exe_unit)
int8_t * varlen_output_buffer_host_ptr_
size_t getCountDistinctDescriptorsSize() const
int64_t allocateCountDistinctSet()
QueryDescriptionType getQueryDescriptionType() const
void compactProjectionBuffersCpu(const QueryMemoryDescriptor &query_mem_desc, const size_t projection_count)
std::shared_ptr< Analyzer::Expr > get_arg1() const
std::vector< int64_t * > group_by_buffers_
void initColumnsPerRow(const QueryMemoryDescriptor &query_mem_desc, int8_t *row_ptr, const std::vector< int64_t > &init_vals, const TargetAggOpsMetadata &agg_op_metadata)
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
void copyGroupByBuffersFromGpu(DeviceAllocator &device_allocator, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit *ra_exe_unit, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer) const
std::optional< size_t > varlenOutputBufferElemSize() const
std::shared_ptr< VarlenOutputInfo > getVarlenOutputInfo()
void initGroupByBuffer(int64_t *buffer, const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, TargetAggOpsMetadata &agg_expr_metadata, const ExecutorDeviceType device_type, const bool output_columnar, const Executor *executor)
size_t bitmapPaddedSizeBytes() const
std::vector< int8_t > get_rows_copy_from_heaps(const int64_t *heaps, const size_t heaps_size, const size_t n, const size_t thread_count)
size_t getNextColOffInBytesRowOnly(const int8_t *col_ptr, const size_t col_idx) const
void allocateTDigestsBuffer(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
void allocateModeBuffer(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
robin_hood::unordered_set< size_t > ModeIndexSet
std::vector< int64_t > get_consistent_frags_sizes(const std::vector< std::vector< uint64_t >> &frag_offsets)
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
CUstream getQueryEngineCudaStreamForDevice(int device_num)
void parallel_for(const blocked_range< Int > &range, const Body &body, const Partitioner &p=Partitioner())
size_t get_heap_size(const size_t row_size, const size_t n, const size_t thread_count)
device_allocator_(device_allocator)
bool interleavedBins(const ExecutorDeviceType) const
const ColSlotContext & getColSlotContext() const
bool threadsShareMemory() const
void copyFromTableFunctionGpuBuffers(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
void applyStreamingTopNOffsetGpu(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit &ra_exe_unit, const unsigned total_thread_count, const int device_id)
const auto getGroupByBuffersSize() const
std::vector< TargetInfo > target_exprs_to_infos(const std::vector< Analyzer::Expr * > &targets, const QueryMemoryDescriptor &query_mem_desc)
void copy_projection_buffer_from_gpu_columnar(Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count, const int device_id)
bool g_optimize_row_initialization
std::shared_ptr< VarlenOutputInfo > varlen_output_info_
int64_t get_consistent_frag_size(const std::vector< uint64_t > &frag_offsets)
int8_t * count_distinct_bitmap_host_crt_ptr_
CUdeviceptr varlen_output_buffer_
std::vector< int64_t > init_agg_val_vec(const std::vector< TargetInfo > &targets, const QueryMemoryDescriptor &query_mem_desc)
const int8_t getSlotIndexForSingleSlotCol(const size_t col_idx) const
size_t getBufferColSlotCount() const
const int8_t getLogicalSlotWidthBytes(const size_t slot_idx) const
size_t getColOffInBytes(const size_t col_idx) const
void copy_group_by_buffers_from_gpu(DeviceAllocator &device_allocator, const std::vector< int64_t * > &group_by_buffers, const size_t groups_buffer_size, const int8_t *group_by_dev_buffers_mem, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer, const bool has_varlen_output)
std::vector< std::unique_ptr< ResultSet > > result_sets_
void initRowGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, TargetAggOpsMetadata &agg_expr_metadata, const int32_t groups_buffer_entry_count, const size_t warp_size, const Executor *executor, const RelAlgExecutionUnit &ra_exe_unit)
virtual void setDeviceMem(int8_t *device_ptr, unsigned char uc, const size_t num_bytes) const =0
void allocateCountDistinctGpuMem(const QueryMemoryDescriptor &query_mem_desc)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
int64_t allocateCountDistinctBitmap(const size_t bitmap_byte_sz)
int get_input_idx(RelAlgExecutionUnit const &ra_exe_unit, const shared::TableKey &outer_table_key)
std::vector< std::vector< int64_t > > get_col_frag_offsets(const std::vector< Analyzer::Expr * > &target_exprs, const std::vector< std::vector< uint64_t >> &table_frag_offsets)