33 const int32_t groups_buffer_entry_count = query_mem_desc.
getEntryCount();
35 const size_t num_count_distinct_descs =
37 for (
size_t i = 0; i < num_count_distinct_descs; i++) {
44 int64_t total_bytes{0};
48 total_bytes =
static_cast<int64_t
>(total_bytes_per_group * groups_buffer_entry_count);
61 const size_t thread_idx,
63 if (render_allocator_map) {
67 const auto gpu_idx = 0;
69 return reinterpret_cast<int64_t*
>(render_allocator_ptr->alloc(numBytes));
71 return reinterpret_cast<int64_t*
>(mem_owner->
allocate(numBytes, thread_idx));
76 if (frag_offsets.size() < 2) {
79 const auto frag_size = frag_offsets[1] - frag_offsets[0];
80 for (
size_t i = 2; i < frag_offsets.size(); ++i) {
81 const auto curr_size = frag_offsets[i] - frag_offsets[i - 1];
82 if (curr_size != frag_size) {
86 return !frag_size ? std::numeric_limits<int64_t>::max()
87 :
static_cast<int64_t
>(frag_size);
91 const std::vector<std::vector<uint64_t>>& frag_offsets) {
92 if (frag_offsets.empty()) {
95 std::vector<int64_t> frag_sizes;
96 for (
size_t tab_idx = 0; tab_idx < frag_offsets[0].size(); ++tab_idx) {
97 std::vector<uint64_t> tab_offs;
98 for (
auto& offsets : frag_offsets) {
99 tab_offs.push_back(offsets[tab_idx]);
107 const std::vector<Analyzer::Expr*>& target_exprs,
108 const std::vector<int64_t>& table_frag_sizes) {
109 std::vector<int64_t> col_frag_sizes;
110 for (
auto expr : target_exprs) {
111 if (
const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
112 if (col_var->get_rte_idx() < 0) {
113 CHECK_EQ(-1, col_var->get_rte_idx());
114 col_frag_sizes.push_back(int64_t(-1));
116 col_frag_sizes.push_back(table_frag_sizes[col_var->get_rte_idx()]);
119 col_frag_sizes.push_back(int64_t(-1));
122 return col_frag_sizes;
126 const std::vector<Analyzer::Expr*>& target_exprs,
127 const std::vector<std::vector<uint64_t>>& table_frag_offsets) {
128 std::vector<std::vector<int64_t>> col_frag_offsets;
129 for (
auto& table_offsets : table_frag_offsets) {
130 std::vector<int64_t> col_offsets;
131 for (
auto expr : target_exprs) {
132 if (
const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
133 if (col_var->get_rte_idx() < 0) {
134 CHECK_EQ(-1, col_var->get_rte_idx());
135 col_offsets.push_back(int64_t(-1));
137 CHECK_LT(static_cast<size_t>(col_var->get_rte_idx()), table_offsets.size());
138 col_offsets.push_back(
139 static_cast<int64_t>(table_offsets[col_var->get_rte_idx()]));
142 col_offsets.push_back(int64_t(-1));
145 col_frag_offsets.push_back(col_offsets);
147 return col_frag_offsets;
154 auto match_table_key = [=](
auto& desc) {
155 return outer_table_key == desc.getTableKey();
158 auto itr = std::find_if(input_descs.begin(), input_descs.end(), match_table_key);
159 return itr == input_descs.end() ? 0 : itr->getNestLevel();
171 const bool output_columnar,
174 const int64_t num_rows,
175 const std::vector<std::vector<const int8_t*>>& col_buffers,
176 const std::vector<std::vector<uint64_t>>& frag_offsets,
179 std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
181 const size_t thread_idx,
182 const Executor* executor)
183 : num_rows_(num_rows)
184 , row_set_mem_owner_(row_set_mem_owner)
185 , init_agg_vals_(executor->plan_state_->init_agg_vals_)
186 ,
num_buffers_(computeNumberOfBuffers(query_mem_desc, device_type, executor))
195 CHECK(!sort_on_gpu || output_columnar);
198 if (consistent_frag_sizes.empty()) {
209 if (render_allocator_map || !query_mem_desc.
isGroupBy()) {
223 ? executor->blockSize() * executor->gridSize()
226 size_t group_buffer_size{0};
231 group_buffer_size = num_rows * query_mem_desc.
getRowSize();
240 CHECK_GE(group_buffer_size,
size_t(0));
243 int64_t* group_by_buffer_template{
nullptr};
244 if (!query_mem_desc.
lazyInitGroups(device_type) && group_buffers_count > 1) {
245 group_by_buffer_template =
reinterpret_cast<int64_t*
>(
262 ? executor->blockSize()
268 const auto actual_group_buffer_size =
269 group_buffer_size + index_buffer_qw *
sizeof(int64_t);
270 CHECK_GE(actual_group_buffer_size, group_buffer_size);
274 CHECK(varlen_buffer_elem_size_opt);
276 query_mem_desc.
getEntryCount() * varlen_buffer_elem_size_opt.value()));
281 for (
size_t i = 0; i < group_buffers_count; i += step) {
283 render_allocator_map,
287 if (group_by_buffer_template) {
288 memcpy(group_by_buffer + index_buffer_qw,
289 group_by_buffer_template,
301 for (
size_t j = 1; j < step; ++j) {
304 const bool use_target_exprs_union =
309 const auto column_frag_sizes =
314 executor->getColLazyFetchInfo(target_exprs),
322 executor->blockSize(),
323 executor->gridSize()));
324 result_sets_.back()->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
325 executor->plan_state_->init_agg_vals_,
327 for (
size_t j = 1; j < step; ++j) {
339 const int64_t num_rows,
340 const std::vector<std::vector<const int8_t*>>& col_buffers,
341 const std::vector<std::vector<uint64_t>>& frag_offsets,
342 std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
344 const Executor* executor)
345 : num_rows_(num_rows)
346 , row_set_mem_owner_(row_set_mem_owner)
359 if (consistent_frag_sizes.empty()) {
364 const size_t num_columns =
366 size_t total_group_by_buffer_size{0};
367 for (
size_t i = 0; i < num_columns; ++i) {
368 auto ti = exe_unit.target_exprs[i]->get_type_info();
373 int64_t flatbuffer_size =
query_mem_desc.getFlatBufferSize(slot_idx);
374 total_group_by_buffer_size =
377 const size_t col_width = ti.get_size();
378 const size_t group_buffer_size = num_rows_ * col_width;
379 total_group_by_buffer_size =
386 total_group_by_buffer_size,
nullptr,
thread_idx_, row_set_mem_owner.get());
387 group_by_buffers_.push_back(group_by_buffer);
389 const auto column_frag_offsets =
391 const auto column_frag_sizes =
393 result_sets_.emplace_back(
403 executor->blockSize(),
404 executor->gridSize()));
405 result_sets_.back()->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
414 const bool output_columnar,
415 const Executor* executor) {
416 if (output_columnar) {
419 auto rows_ptr = buffer;
422 ? executor->blockSize() * executor->gridSize()
425 query_mem_desc.
interleavedBins(device_type) ? executor->warpSize() : 1;
427 const auto node_count_size = thread_count *
sizeof(int64_t);
428 memset(rows_ptr, 0, node_count_size);
431 memset(rows_ptr + thread_count, -1, rows_offset - node_count_size);
432 rows_ptr += rows_offset /
sizeof(int64_t);
433 actual_entry_count =
n * thread_count;
446 int64_t* groups_buffer,
447 const std::vector<int64_t>& init_vals,
448 const int32_t groups_buffer_entry_count,
449 const size_t warp_size,
450 const Executor* executor) {
452 const size_t row_size{query_mem_desc.
getRowSize()};
456 auto mode_index_set =
allocateModes(query_mem_desc,
true, executor);
458 auto buffer_ptr =
reinterpret_cast<int8_t*
>(groups_buffer);
460 const auto query_mem_desc_fixedup =
463 auto const is_true = [](
auto const& x) {
return static_cast<bool>(x); };
466 if (!
std::any_of(agg_bitmap_size.begin(), agg_bitmap_size.end(), is_true) &&
467 !
std::any_of(quantile_params.begin(), quantile_params.end(), is_true) &&
469 std::vector<int8_t> sample_row(row_size - col_base_off);
479 CHECK(warp_size >= 1);
480 CHECK(key_count == 1 || warp_size == 1);
481 for (
size_t warp_idx = 0; warp_idx < warp_size; ++warp_idx) {
482 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
483 ++bin, buffer_ptr += row_size) {
484 memcpy(buffer_ptr + col_base_off, sample_row.data(), sample_row.size());
490 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
491 ++bin, buffer_ptr += row_size) {
492 memcpy(buffer_ptr + col_base_off, sample_row.data(), sample_row.size());
498 CHECK(warp_size >= 1);
499 CHECK(key_count == 1 || warp_size == 1);
500 for (
size_t warp_idx = 0; warp_idx < warp_size; ++warp_idx) {
501 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
502 ++bin, buffer_ptr += row_size) {
504 &buffer_ptr[col_base_off],
514 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
515 ++bin, buffer_ptr += row_size) {
519 &buffer_ptr[col_base_off],
530 template <
typename T>
532 static_assert(
sizeof(
T) <=
sizeof(int64_t),
"Unsupported template type");
533 for (uint32_t i = 0; i < entry_count; ++i) {
534 buffer_ptr[i] = init_val;
536 return reinterpret_cast<int8_t*
>(buffer_ptr + entry_count);
543 int64_t* groups_buffer,
544 const std::vector<int64_t>& init_vals,
545 const Executor* executor) {
546 CHECK(groups_buffer);
548 for (
const auto target_expr : executor->plan_state_->target_exprs_) {
552 const int32_t agg_col_count = query_mem_desc.
getSlotCount();
553 auto buffer_ptr =
reinterpret_cast<int8_t*
>(groups_buffer);
555 const auto groups_buffer_entry_count = query_mem_desc.
getEntryCount();
558 for (
size_t i = 0; i < key_count; ++i) {
559 buffer_ptr = initColumnarBuffer<int64_t>(
reinterpret_cast<int64_t*
>(buffer_ptr),
561 groups_buffer_entry_count);
567 int32_t init_val_idx = 0;
568 for (int32_t i = 0; i < agg_col_count; ++i) {
570 CHECK_LT(static_cast<size_t>(init_val_idx), init_vals.size());
573 buffer_ptr = initColumnarBuffer<int8_t>(
574 buffer_ptr, init_vals[init_val_idx++], groups_buffer_entry_count);
578 initColumnarBuffer<int16_t>(
reinterpret_cast<int16_t*
>(buffer_ptr),
579 init_vals[init_val_idx++],
580 groups_buffer_entry_count);
584 initColumnarBuffer<int32_t>(
reinterpret_cast<int32_t*
>(buffer_ptr),
585 init_vals[init_val_idx++],
586 groups_buffer_entry_count);
590 initColumnarBuffer<int64_t>(
reinterpret_cast<int64_t*
>(buffer_ptr),
591 init_vals[init_val_idx++],
592 groups_buffer_entry_count);
609 const std::vector<int64_t>& init_vals,
610 const std::vector<int64_t>& bitmap_sizes,
612 const std::vector<QuantileParam>& quantile_params) {
613 int8_t* col_ptr = row_ptr;
614 size_t init_vec_idx = 0;
615 for (
size_t col_idx = 0; col_idx < query_mem_desc.
getSlotCount();
617 const int64_t bm_sz{bitmap_sizes[col_idx]};
619 if (bm_sz && query_mem_desc.
isGroupBy()) {
626 }
else if (query_mem_desc.
isGroupBy() && quantile_params[col_idx]) {
627 auto const q = *quantile_params[col_idx];
631 }
else if (query_mem_desc.
isGroupBy() && mode_index_set.count(col_idx)) {
636 CHECK_LT(init_vec_idx, init_vals.size());
637 init_val = init_vals[init_vec_idx++];
642 *col_ptr =
static_cast<int8_t
>(init_val);
645 *
reinterpret_cast<int16_t*
>(col_ptr) = (int16_t)init_val;
648 *
reinterpret_cast<int32_t*
>(col_ptr) = (int32_t)init_val;
651 *
reinterpret_cast<int64_t*
>(col_ptr) = init_val;
668 size_t total_bytes_per_entry{0};
669 const size_t num_count_distinct_descs =
671 for (
size_t i = 0; i < num_count_distinct_descs; i++) {
677 total_bytes_per_entry += count_distinct_desc.bitmapPaddedSizeBytes();
696 const Executor* executor) {
697 const size_t agg_col_count{query_mem_desc.
getSlotCount()};
698 std::vector<int64_t> agg_bitmap_size(deferred ? agg_col_count : 0);
700 CHECK_GE(agg_col_count, executor->plan_state_->target_exprs_.size());
701 for (
size_t target_idx = 0; target_idx < executor->plan_state_->target_exprs_.size();
703 const auto target_expr = executor->plan_state_->target_exprs_[target_idx];
706 CHECK(agg_info.is_agg &&
709 CHECK(!agg_info.sql_type.is_varlen());
712 CHECK_LT(static_cast<size_t>(agg_col_idx), agg_col_count);
716 const auto& count_distinct_desc =
720 const auto bitmap_byte_sz = count_distinct_desc.bitmapPaddedSizeBytes();
722 agg_bitmap_size[agg_col_idx] = bitmap_byte_sz;
729 agg_bitmap_size[agg_col_idx] = -1;
737 return agg_bitmap_size;
746 ptr, bitmap_byte_sz,
false);
747 return reinterpret_cast<int64_t
>(ptr);
749 return reinterpret_cast<int64_t
>(
756 return reinterpret_cast<int64_t
>(count_distinct_set);
762 std::vector<Analyzer::Expr*>
const& target_exprs,
765 for (
size_t target_idx = 0; target_idx < target_exprs.size(); ++target_idx) {
766 auto const target_expr = target_exprs[target_idx];
767 if (
auto const* agg_expr = dynamic_cast<Analyzer::AggExpr const*>(target_expr)) {
768 if (agg_expr->get_aggtype() == agg_type) {
769 lambda(agg_expr, target_idx);
780 const Executor* executor) {
781 size_t const slot_count = query_mem_desc.
getSlotCount();
782 CHECK_LE(executor->plan_state_->target_exprs_.size(), slot_count);
786 executor->plan_state_->target_exprs_,
789 size_t const agg_col_idx =
793 mode_index_set.emplace(agg_col_idx);
796 init_agg_vals_[agg_col_idx] =
reinterpret_cast<int64_t
>(agg_mode);
799 return mode_index_set;
802 std::vector<QueryMemoryInitializer::QuantileParam>
805 const Executor* executor) {
806 size_t const slot_count = query_mem_desc.
getSlotCount();
807 CHECK_LE(executor->plan_state_->target_exprs_.size(), slot_count);
808 std::vector<QuantileParam> quantile_params(deferred ? slot_count : 0);
811 executor->plan_state_->target_exprs_,
814 size_t const agg_col_idx =
817 CHECK_EQ(static_cast<int8_t>(
sizeof(int64_t)),
822 auto const q = q_expr->get_constval().doubleval;
824 quantile_params[agg_col_idx] = q;
831 return quantile_params;
836 const int8_t* init_agg_vals_dev_ptr,
839 const unsigned block_size_x,
840 const unsigned grid_size_x) {
843 const auto thread_count = block_size_x * grid_size_x;
844 const auto total_buff_size =
848 std::vector<int8_t*> dev_buffers(thread_count);
850 for (
size_t i = 0; i < thread_count; ++i) {
851 dev_buffers[i] = dev_buffer;
856 dev_ptr, dev_buffers.data(), thread_count *
sizeof(int8_t*));
861 thread_count *
sizeof(int64_t));
864 reinterpret_cast<int8_t*>(dev_buffer + thread_count *
sizeof(int64_t)),
866 thread_count * n *
sizeof(int64_t));
869 reinterpret_cast<int64_t*>(
871 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
875 query_mem_desc.
getRowSize() /
sizeof(int64_t),
881 return {dev_ptr, dev_buffer};
891 const int8_t* init_agg_vals_dev_ptr,
894 const unsigned block_size_x,
895 const unsigned grid_size_x,
896 const int8_t warp_size,
897 const bool can_sort_on_gpu,
898 const bool output_columnar,
902 if (render_allocator) {
906 CHECK(!output_columnar);
909 query_mem_desc, init_agg_vals_dev_ptr,
n, device_id, block_size_x, grid_size_x);
912 auto dev_group_by_buffers =
927 CHECK(dev_group_by_buffers.varlen_output_buffer);
929 reinterpret_cast<CUdeviceptr>(dev_group_by_buffers.varlen_output_buffer);
931 const size_t varlen_output_buf_bytes =
940 if (render_allocator) {
944 CHECK(!render_allocator);
949 auto group_by_dev_buffer = dev_group_by_buffers.data;
951 int8_t* col_widths_dev_ptr{
nullptr};
952 if (output_columnar) {
953 std::vector<int8_t> compact_col_widths(col_count);
954 for (
size_t idx = 0; idx < col_count; ++idx) {
959 col_widths_dev_ptr, compact_col_widths.data(), col_count *
sizeof(int8_t));
961 const int8_t warp_count =
963 const auto num_group_by_buffers =
965 for (
size_t i = 0; i < num_group_by_buffers; i += step) {
966 if (output_columnar) {
968 reinterpret_cast<int64_t*>(group_by_dev_buffer),
969 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
970 dev_group_by_buffers.entry_count,
981 reinterpret_cast<int64_t*>(group_by_dev_buffer),
982 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
983 dev_group_by_buffers.entry_count,
986 query_mem_desc.
getRowSize() /
sizeof(int64_t),
992 group_by_dev_buffer += groups_buffer_size;
995 return dev_group_by_buffers;
1004 const int device_id,
1005 const unsigned block_size_x,
1006 const unsigned grid_size_x,
1007 const bool zero_initialize_buffers) {
1010 size_t total_group_by_buffer_size{0};
1013 std::vector<size_t> col_byte_offsets;
1014 col_byte_offsets.reserve(num_columns);
1016 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1018 size_t group_buffer_size =
num_rows_ * col_width;
1019 col_byte_offsets.emplace_back(total_group_by_buffer_size);
1020 total_group_by_buffer_size =
1024 int8_t* dev_buffers_allocation{
nullptr};
1026 CHECK(dev_buffers_allocation);
1027 if (zero_initialize_buffers) {
1031 auto dev_buffers_mem = dev_buffers_allocation;
1032 std::vector<int8_t*> dev_buffers(num_columns);
1033 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1034 dev_buffers[col_idx] = dev_buffers_allocation + col_byte_offsets[col_idx];
1038 dev_ptrs, dev_buffers.data(), num_columns *
sizeof(
CUdeviceptr));
1040 return {dev_ptrs, dev_buffers_mem, (size_t)
num_rows_};
1046 const size_t entry_count,
1048 const int device_id,
1049 const unsigned block_size_x,
1050 const unsigned grid_size_x) {
1053 int8_t* dev_buffer = gpu_group_by_buffers.
data;
1056 const size_t original_entry_count = gpu_group_by_buffers.
entry_count;
1057 CHECK_LE(entry_count, original_entry_count);
1058 size_t output_device_col_offset{0};
1059 size_t output_host_col_offset{0};
1063 auto allocator = std::make_unique<CudaAllocator>(
1066 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1068 const size_t output_device_col_size = original_entry_count * col_width;
1069 const size_t output_host_col_size = entry_count * col_width;
1070 allocator->copyFromDevice(host_buffer + output_host_col_offset,
1071 dev_buffer + output_device_col_offset,
1072 output_host_col_size);
1073 output_device_col_offset =
1074 align_to_int64(output_device_col_offset + output_device_col_size);
1075 output_host_col_offset =
1086 : executor->blockSize() *
1095 int8_t* projection_buffer,
1096 const size_t projection_count) {
1099 constexpr
size_t row_index_width =
sizeof(int64_t);
1100 size_t buffer_offset1{projection_count * row_index_width};
1102 for (
size_t i = 0; i < query_mem_desc.
getSlotCount(); i++) {
1104 auto column_proj_size =
1107 if (buffer_offset1 + column_proj_size >= buffer_offset2) {
1109 std::memmove(projection_buffer + buffer_offset1,
1110 projection_buffer + buffer_offset2,
1113 std::memcpy(projection_buffer + buffer_offset1,
1114 projection_buffer + buffer_offset2,
1126 const size_t projection_count) {
1127 const auto num_allocated_rows =
1129 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1135 num_allocated_rows);
1139 result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1146 const size_t projection_count,
1147 const int device_id) {
1149 const auto num_allocated_rows =
1153 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1156 gpu_group_by_buffers,
1164 result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1170 const size_t entry_count,
1173 const unsigned block_size_x,
1174 const unsigned grid_size_x,
1175 const int device_id,
1176 const bool prepend_index_buffer)
const {
1177 const auto thread_count = block_size_x * grid_size_x;
1179 size_t total_buff_size{0};
1191 gpu_group_by_buffers.
data,
1196 prepend_index_buffer,
1203 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1221 const unsigned total_thread_count,
1222 const int device_id) {
1225 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1227 const auto rows_copy = pick_top_n_rows_from_dev_heaps(
1229 reinterpret_cast<int64_t*>(gpu_group_by_buffers.
data),
GpuGroupByBuffers setupTableFunctionGpuBuffers(const QueryMemoryDescriptor &query_mem_desc, const int device_id, const unsigned block_size_x, const unsigned grid_size_x, const bool zero_initialize_buffers)
std::vector< Analyzer::Expr * > target_exprs
size_t getSlotCount() const
size_t getBufferSizeBytes(const RelAlgExecutionUnit &ra_exe_unit, const unsigned thread_count, const ExecutorDeviceType device_type) const
GpuGroupByBuffers create_dev_group_by_buffers(DeviceAllocator *device_allocator, const std::vector< int64_t * > &group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const ExecutorDispatchMode dispatch_mode, const int64_t num_input_rows, const bool prepend_index_buffer, const bool always_init_group_by_on_host, const bool use_bump_allocator, const bool has_varlen_output, Allocator *insitu_allocator)
RenderAllocator * getRenderAllocator(size_t device_id)
ModeIndexSet allocateModes(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
robin_hood::unordered_set< int64_t > CountDistinctSet
bool countDistinctDescriptorsLogicallyEmpty() const
size_t getEntryCount() const
bool useCudaBuffers() const
GpuGroupByBuffers prepareTopNHeapsDevBuffer(const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const size_t n, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
size_t count_distinct_bitmap_mem_bytes_
void initColumnsPerRow(const QueryMemoryDescriptor &query_mem_desc, int8_t *row_ptr, const std::vector< int64_t > &init_vals, const std::vector< int64_t > &bitmap_sizes, const ModeIndexSet &mode_index_set, const std::vector< QuantileParam > &quantile_params)
GpuGroupByBuffers createAndInitializeGroupByBufferGpu(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const int device_id, const ExecutorDispatchMode dispatch_mode, const unsigned block_size_x, const unsigned grid_size_x, const int8_t warp_size, const bool can_sort_on_gpu, const bool output_columnar, RenderAllocator *render_allocator)
boost::multiprecision::number< boost::multiprecision::cpp_int_backend< 64, 64, boost::multiprecision::signed_magnitude, boost::multiprecision::checked, void >> checked_int64_t
void compact_projection_buffer_for_cpu_columnar(const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count)
DeviceAllocator * device_allocator_
void sort_on_gpu(int64_t *val_buff, int32_t *idx_buff, const uint64_t entry_count, const bool desc, const uint32_t chosen_bytes, ThrustAllocator &alloc, const int device_id)
const std::optional< bool > union_all
Streaming Top N algorithm.
size_t get_rows_offset_of_heaps(const size_t n, const size_t thread_count)
bool hasVarlenOutput() const
QueryMemoryInitializer(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int device_id, const ExecutorDeviceType device_type, const ExecutorDispatchMode dispatch_mode, const bool output_columnar, const bool sort_on_gpu, const shared::TableKey &outer_table_key, const int64_t num_rows, const std::vector< std::vector< const int8_t * >> &col_buffers, const std::vector< std::vector< uint64_t >> &frag_offsets, RenderAllocatorMap *render_allocator_map, RenderInfo *render_info, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, DeviceAllocator *gpu_allocator, const size_t thread_idx, const Executor *executor)
std::vector< int64_t > allocateCountDistinctBuffers(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
unsigned long long CUdeviceptr
int8_t * allocate(const size_t num_bytes, const size_t thread_idx=0) override
std::vector< InputDescriptor > input_descs
bool hasKeylessHash() const
void init_columnar_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t agg_col_count, const int8_t *col_sizes, const bool need_padding, const bool keyless, const int8_t key_size, const size_t block_size_x, const size_t grid_size_x)
CUdeviceptr count_distinct_bitmap_mem_
void check_total_bitmap_memory(const QueryMemoryDescriptor &query_mem_desc)
virtual int8_t * alloc(const size_t num_bytes)=0
size_t getEffectiveKeyWidth() const
void eachAggregateTargetIdxOfType(std::vector< Analyzer::Expr * > const &target_exprs, SQLAgg const agg_type, std::function< void(Analyzer::AggExpr const *, size_t)> lambda)
int8_t * initColumnarBuffer(T *buffer_ptr, const T init_val, const uint32_t entry_count)
TargetInfo get_target_info(const Analyzer::Expr *target_expr, const bool bigint_count)
size_t computeNumberOfBuffers(const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const Executor *executor) const
std::vector< QuantileParam > allocateTDigests(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
size_t getRowSize() const
varlen_output_buffer_host_ptr_(nullptr)
void init_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad, const bool keyless, const int8_t warp_size, const size_t block_size_x, const size_t grid_size_x)
const SlotSize & getSlotInfo(const size_t slot_idx) const
std::vector< Analyzer::Expr * > target_exprs_union
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
void compactProjectionBuffersGpu(const QueryMemoryDescriptor &query_mem_desc, Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const size_t projection_count, const int device_id)
bool blocksShareMemory() const
virtual void copyToDevice(void *device_dst, const void *host_src, const size_t num_bytes) const =0
std::vector< int64_t > init_agg_vals_
bool useStreamingTopN() const
size_t getGroupbyColCount() const
void applyStreamingTopNOffsetCpu(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
void fill_empty_key(void *key_ptr, const size_t key_count, const size_t key_width)
virtual void zeroDeviceMem(int8_t *device_ptr, const size_t num_bytes) const =0
bool lazyInitGroups(const ExecutorDeviceType) const
int64_t g_bitmap_memory_limit
size_t g_max_memory_allocation_size
size_t getAllocatedSize() const
bool is_distinct_target(const TargetInfo &target_info)
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
int8_t * count_distinct_bitmap_crt_ptr_
const std::shared_ptr< Analyzer::Estimator > estimator
void initGroupByBuffer(int64_t *buffer, const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const bool output_columnar, const Executor *executor)
int8_t * varlen_output_buffer_host_ptr_
size_t getCountDistinctDescriptorsSize() const
int64_t allocateCountDistinctSet()
QueryDescriptionType getQueryDescriptionType() const
void compactProjectionBuffersCpu(const QueryMemoryDescriptor &query_mem_desc, const size_t projection_count)
std::shared_ptr< Analyzer::Expr > get_arg1() const
std::vector< int64_t * > group_by_buffers_
void initColumnarGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const Executor *executor)
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
void copyGroupByBuffersFromGpu(DeviceAllocator &device_allocator, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit *ra_exe_unit, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer) const
std::optional< size_t > varlenOutputBufferElemSize() const
std::shared_ptr< VarlenOutputInfo > getVarlenOutputInfo()
count_distinct_bitmap_mem_(0)
size_t bitmapPaddedSizeBytes() const
std::vector< int8_t > get_rows_copy_from_heaps(const int64_t *heaps, const size_t heaps_size, const size_t n, const size_t thread_count)
size_t getNextColOffInBytesRowOnly(const int8_t *col_ptr, const size_t col_idx) const
robin_hood::unordered_set< size_t > ModeIndexSet
std::vector< int64_t > get_consistent_frags_sizes(const std::vector< std::vector< uint64_t >> &frag_offsets)
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
CUstream getQueryEngineCudaStreamForDevice(int device_num)
count_distinct_bitmap_host_mem_(nullptr)
size_t get_heap_size(const size_t row_size, const size_t n, const size_t thread_count)
device_allocator_(device_allocator)
bool interleavedBins(const ExecutorDeviceType) const
const ColSlotContext & getColSlotContext() const
bool threadsShareMemory() const
void copyFromTableFunctionGpuBuffers(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
void applyStreamingTopNOffsetGpu(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit &ra_exe_unit, const unsigned total_thread_count, const int device_id)
const auto getGroupByBuffersSize() const
std::vector< TargetInfo > target_exprs_to_infos(const std::vector< Analyzer::Expr * > &targets, const QueryMemoryDescriptor &query_mem_desc)
void copy_projection_buffer_from_gpu_columnar(Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count, const int device_id)
bool g_optimize_row_initialization
bool any_of(std::vector< Analyzer::Expr * > const &target_exprs)
count_distinct_bitmap_crt_ptr_(nullptr)
std::shared_ptr< VarlenOutputInfo > varlen_output_info_
int64_t get_consistent_frag_size(const std::vector< uint64_t > &frag_offsets)
int8_t * count_distinct_bitmap_host_mem_
count_distinct_bitmap_mem_bytes_(0)
CUdeviceptr varlen_output_buffer_
std::vector< int64_t > init_agg_val_vec(const std::vector< TargetInfo > &targets, const QueryMemoryDescriptor &query_mem_desc)
const int8_t getSlotIndexForSingleSlotCol(const size_t col_idx) const
size_t getBufferColSlotCount() const
const int8_t getLogicalSlotWidthBytes(const size_t slot_idx) const
size_t getColOffInBytes(const size_t col_idx) const
int64_t * alloc_group_by_buffer(const size_t numBytes, RenderAllocatorMap *render_allocator_map, const size_t thread_idx, RowSetMemoryOwner *mem_owner)
void copy_group_by_buffers_from_gpu(DeviceAllocator &device_allocator, const std::vector< int64_t * > &group_by_buffers, const size_t groups_buffer_size, const int8_t *group_by_dev_buffers_mem, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer, const bool has_varlen_output)
std::vector< std::unique_ptr< ResultSet > > result_sets_
virtual void setDeviceMem(int8_t *device_ptr, unsigned char uc, const size_t num_bytes) const =0
void allocateCountDistinctGpuMem(const QueryMemoryDescriptor &query_mem_desc)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
int64_t allocateCountDistinctBitmap(const size_t bitmap_byte_sz)
void initRowGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const int32_t groups_buffer_entry_count, const size_t warp_size, const Executor *executor)
int get_input_idx(RelAlgExecutionUnit const &ra_exe_unit, const shared::TableKey &outer_table_key)
std::vector< std::vector< int64_t > > get_col_frag_offsets(const std::vector< Analyzer::Expr * > &target_exprs, const std::vector< std::vector< uint64_t >> &table_frag_offsets)