37 const int32_t groups_buffer_entry_count = query_mem_desc.
getEntryCount();
39 const size_t num_count_distinct_descs =
41 for (
size_t i = 0; i < num_count_distinct_descs; i++) {
48 int64_t total_bytes{0};
52 total_bytes =
static_cast<int64_t
>(total_bytes_per_group * groups_buffer_entry_count);
65 const size_t thread_idx,
67 if (render_allocator_map) {
71 const auto gpu_idx = 0;
73 return reinterpret_cast<int64_t*
>(render_allocator_ptr->alloc(numBytes));
75 return reinterpret_cast<int64_t*
>(mem_owner->
allocate(numBytes, thread_idx));
80 if (frag_offsets.size() < 2) {
83 const auto frag_size = frag_offsets[1] - frag_offsets[0];
84 for (
size_t i = 2; i < frag_offsets.size(); ++i) {
85 const auto curr_size = frag_offsets[i] - frag_offsets[i - 1];
86 if (curr_size != frag_size) {
90 return !frag_size ? std::numeric_limits<int64_t>::max()
91 :
static_cast<int64_t
>(frag_size);
95 const std::vector<std::vector<uint64_t>>& frag_offsets) {
96 if (frag_offsets.empty()) {
99 std::vector<int64_t> frag_sizes;
100 for (
size_t tab_idx = 0; tab_idx < frag_offsets[0].size(); ++tab_idx) {
101 std::vector<uint64_t> tab_offs;
102 for (
auto& offsets : frag_offsets) {
103 tab_offs.push_back(offsets[tab_idx]);
111 const std::vector<Analyzer::Expr*>& target_exprs,
112 const std::vector<int64_t>& table_frag_sizes) {
113 std::vector<int64_t> col_frag_sizes;
114 for (
auto expr : target_exprs) {
115 if (
const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
116 if (col_var->get_rte_idx() < 0) {
117 CHECK_EQ(-1, col_var->get_rte_idx());
118 col_frag_sizes.push_back(int64_t(-1));
120 col_frag_sizes.push_back(table_frag_sizes[col_var->get_rte_idx()]);
123 col_frag_sizes.push_back(int64_t(-1));
126 return col_frag_sizes;
130 const std::vector<Analyzer::Expr*>& target_exprs,
131 const std::vector<std::vector<uint64_t>>& table_frag_offsets) {
132 std::vector<std::vector<int64_t>> col_frag_offsets;
133 for (
auto& table_offsets : table_frag_offsets) {
134 std::vector<int64_t> col_offsets;
135 for (
auto expr : target_exprs) {
136 if (
const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
137 if (col_var->get_rte_idx() < 0) {
138 CHECK_EQ(-1, col_var->get_rte_idx());
139 col_offsets.push_back(int64_t(-1));
141 CHECK_LT(static_cast<size_t>(col_var->get_rte_idx()), table_offsets.size());
142 col_offsets.push_back(
143 static_cast<int64_t>(table_offsets[col_var->get_rte_idx()]));
146 col_offsets.push_back(int64_t(-1));
149 col_frag_offsets.push_back(col_offsets);
151 return col_frag_offsets;
157 auto match_table_id = [=](
auto& desc) {
return outer_table_id == desc.getTableId(); };
159 auto itr = std::find_if(input_descs.begin(), input_descs.end(), match_table_id);
160 return itr == input_descs.end() ? 0 : itr->getNestLevel();
172 const bool output_columnar,
174 const int outer_table_id,
175 const int64_t num_rows,
176 const std::vector<std::vector<const int8_t*>>& col_buffers,
177 const std::vector<std::vector<uint64_t>>& frag_offsets,
180 std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
182 const size_t thread_idx,
183 const Executor* executor)
184 : num_rows_(num_rows)
185 , row_set_mem_owner_(row_set_mem_owner)
186 , init_agg_vals_(executor->plan_state_->init_agg_vals_)
187 ,
num_buffers_(computeNumberOfBuffers(query_mem_desc, device_type, executor))
196 CHECK(!sort_on_gpu || output_columnar);
199 if (consistent_frag_sizes.empty()) {
210 if (render_allocator_map || !query_mem_desc.
isGroupBy()) {
223 ? executor->blockSize() * executor->gridSize()
226 size_t group_buffer_size{0};
231 group_buffer_size = num_rows * query_mem_desc.
getRowSize();
240 CHECK_GE(group_buffer_size,
size_t(0));
243 int64_t* group_by_buffer_template{
nullptr};
244 if (!query_mem_desc.
lazyInitGroups(device_type) && group_buffers_count > 1) {
245 group_by_buffer_template =
reinterpret_cast<int64_t*
>(
262 ? executor->blockSize()
268 const auto actual_group_buffer_size =
269 group_buffer_size + index_buffer_qw *
sizeof(int64_t);
270 CHECK_GE(actual_group_buffer_size, group_buffer_size);
274 CHECK(varlen_buffer_elem_size_opt);
276 query_mem_desc.
getEntryCount() * varlen_buffer_elem_size_opt.value()));
281 for (
size_t i = 0; i < group_buffers_count; i += step) {
283 render_allocator_map,
287 if (group_by_buffer_template) {
288 memcpy(group_by_buffer + index_buffer_qw,
289 group_by_buffer_template,
301 for (
size_t j = 1; j < step; ++j) {
304 const bool use_target_exprs_union =
309 const auto column_frag_sizes =
314 executor->getColLazyFetchInfo(target_exprs),
322 executor->getCatalog(),
323 executor->blockSize(),
324 executor->gridSize()));
325 result_sets_.back()->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
326 executor->plan_state_->init_agg_vals_,
328 for (
size_t j = 1; j < step; ++j) {
340 const int64_t num_rows,
341 const std::vector<std::vector<const int8_t*>>& col_buffers,
342 const std::vector<std::vector<uint64_t>>& frag_offsets,
343 std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
345 const Executor* executor)
346 : num_rows_(num_rows)
347 , row_set_mem_owner_(row_set_mem_owner)
360 if (consistent_frag_sizes.empty()) {
365 const size_t num_columns =
query_mem_desc.getBufferColSlotCount();
366 size_t total_group_by_buffer_size{0};
367 for (
size_t i = 0; i < num_columns; ++i) {
368 const size_t col_width = exe_unit.target_exprs[i]->get_type_info().get_size();
369 const size_t group_buffer_size = num_rows_ * col_width;
370 total_group_by_buffer_size =
376 total_group_by_buffer_size,
nullptr,
thread_idx_, row_set_mem_owner.get());
377 group_by_buffers_.push_back(group_by_buffer);
379 const auto column_frag_offsets =
381 const auto column_frag_sizes =
383 result_sets_.emplace_back(
393 executor->getCatalog(),
394 executor->blockSize(),
395 executor->gridSize()));
396 result_sets_.back()->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
405 const bool output_columnar,
406 const Executor* executor) {
407 if (output_columnar) {
410 auto rows_ptr = buffer;
413 ? executor->blockSize() * executor->gridSize()
416 query_mem_desc.
interleavedBins(device_type) ? executor->warpSize() : 1;
418 const auto node_count_size = thread_count *
sizeof(int64_t);
419 memset(rows_ptr, 0, node_count_size);
422 memset(rows_ptr + thread_count, -1, rows_offset - node_count_size);
423 rows_ptr += rows_offset /
sizeof(int64_t);
424 actual_entry_count =
n * thread_count;
437 int64_t* groups_buffer,
438 const std::vector<int64_t>& init_vals,
439 const int32_t groups_buffer_entry_count,
440 const size_t warp_size,
441 const Executor* executor) {
443 const size_t row_size{query_mem_desc.
getRowSize()};
448 auto buffer_ptr =
reinterpret_cast<int8_t*
>(groups_buffer);
450 const auto query_mem_desc_fixedup =
453 auto const is_true = [](
auto const& x) {
return static_cast<bool>(x); };
456 if (!std::any_of(agg_bitmap_size.begin(), agg_bitmap_size.end(), is_true) &&
457 !std::any_of(quantile_params.begin(), quantile_params.end(), is_true) &&
459 std::vector<int8_t> sample_row(row_size - col_base_off);
468 CHECK(warp_size >= 1);
469 CHECK(key_count == 1 || warp_size == 1);
470 for (
size_t warp_idx = 0; warp_idx < warp_size; ++warp_idx) {
471 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
472 ++bin, buffer_ptr += row_size) {
473 memcpy(buffer_ptr + col_base_off, sample_row.data(), sample_row.size());
479 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
480 ++bin, buffer_ptr += row_size) {
481 memcpy(buffer_ptr + col_base_off, sample_row.data(), sample_row.size());
487 CHECK(warp_size >= 1);
488 CHECK(key_count == 1 || warp_size == 1);
489 for (
size_t warp_idx = 0; warp_idx < warp_size; ++warp_idx) {
490 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
491 ++bin, buffer_ptr += row_size) {
493 &buffer_ptr[col_base_off],
502 for (
size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
503 ++bin, buffer_ptr += row_size) {
507 &buffer_ptr[col_base_off],
517 template <
typename T>
519 static_assert(
sizeof(
T) <=
sizeof(int64_t),
"Unsupported template type");
520 for (uint32_t i = 0; i < entry_count; ++i) {
521 buffer_ptr[i] = init_val;
523 return reinterpret_cast<int8_t*
>(buffer_ptr + entry_count);
530 int64_t* groups_buffer,
531 const std::vector<int64_t>& init_vals,
532 const Executor* executor) {
533 CHECK(groups_buffer);
535 for (
const auto target_expr : executor->plan_state_->target_exprs_) {
539 const int32_t agg_col_count = query_mem_desc.
getSlotCount();
540 auto buffer_ptr =
reinterpret_cast<int8_t*
>(groups_buffer);
542 const auto groups_buffer_entry_count = query_mem_desc.
getEntryCount();
545 for (
size_t i = 0; i < key_count; ++i) {
546 buffer_ptr = initColumnarBuffer<int64_t>(
reinterpret_cast<int64_t*
>(buffer_ptr),
548 groups_buffer_entry_count);
554 int32_t init_val_idx = 0;
555 for (int32_t i = 0; i < agg_col_count; ++i) {
557 CHECK_LT(static_cast<size_t>(init_val_idx), init_vals.size());
560 buffer_ptr = initColumnarBuffer<int8_t>(
561 buffer_ptr, init_vals[init_val_idx++], groups_buffer_entry_count);
565 initColumnarBuffer<int16_t>(
reinterpret_cast<int16_t*
>(buffer_ptr),
566 init_vals[init_val_idx++],
567 groups_buffer_entry_count);
571 initColumnarBuffer<int32_t>(
reinterpret_cast<int32_t*
>(buffer_ptr),
572 init_vals[init_val_idx++],
573 groups_buffer_entry_count);
577 initColumnarBuffer<int64_t>(
reinterpret_cast<int64_t*
>(buffer_ptr),
578 init_vals[init_val_idx++],
579 groups_buffer_entry_count);
596 const std::vector<int64_t>& init_vals,
597 const std::vector<int64_t>& bitmap_sizes,
598 const std::vector<QuantileParam>& quantile_params) {
599 int8_t* col_ptr = row_ptr;
600 size_t init_vec_idx = 0;
601 for (
size_t col_idx = 0; col_idx < query_mem_desc.
getSlotCount();
603 const int64_t bm_sz{bitmap_sizes[col_idx]};
605 if (bm_sz && query_mem_desc.
isGroupBy()) {
612 }
else if (query_mem_desc.
isGroupBy() && quantile_params[col_idx]) {
613 auto const q = *quantile_params[col_idx];
619 CHECK_LT(init_vec_idx, init_vals.size());
620 init_val = init_vals[init_vec_idx++];
625 *col_ptr =
static_cast<int8_t
>(init_val);
628 *
reinterpret_cast<int16_t*
>(col_ptr) = (int16_t)init_val;
631 *
reinterpret_cast<int32_t*
>(col_ptr) = (int32_t)init_val;
634 *
reinterpret_cast<int64_t*
>(col_ptr) = init_val;
651 size_t total_bytes_per_entry{0};
652 const size_t num_count_distinct_descs =
654 for (
size_t i = 0; i < num_count_distinct_descs; i++) {
660 total_bytes_per_entry += count_distinct_desc.bitmapPaddedSizeBytes();
679 const Executor* executor) {
680 const size_t agg_col_count{query_mem_desc.
getSlotCount()};
681 std::vector<int64_t> agg_bitmap_size(deferred ? agg_col_count : 0);
683 CHECK_GE(agg_col_count, executor->plan_state_->target_exprs_.size());
684 for (
size_t target_idx = 0; target_idx < executor->plan_state_->target_exprs_.size();
686 const auto target_expr = executor->plan_state_->target_exprs_[target_idx];
689 CHECK(agg_info.is_agg &&
691 CHECK(!agg_info.sql_type.is_varlen());
694 CHECK_LT(static_cast<size_t>(agg_col_idx), agg_col_count);
698 const auto& count_distinct_desc =
702 const auto bitmap_byte_sz = count_distinct_desc.bitmapPaddedSizeBytes();
704 agg_bitmap_size[agg_col_idx] = bitmap_byte_sz;
711 agg_bitmap_size[agg_col_idx] = -1;
719 return agg_bitmap_size;
728 ptr, bitmap_byte_sz,
false);
729 return reinterpret_cast<int64_t
>(ptr);
731 return reinterpret_cast<int64_t
>(
738 return reinterpret_cast<int64_t
>(count_distinct_set);
741 std::vector<QueryMemoryInitializer::QuantileParam>
744 const Executor* executor) {
745 size_t const slot_count = query_mem_desc.
getSlotCount();
746 size_t const ntargets = executor->plan_state_->target_exprs_.size();
748 std::vector<QuantileParam> quantile_params(deferred ? slot_count : 0);
750 for (
size_t target_idx = 0; target_idx < ntargets; ++target_idx) {
751 auto const target_expr = executor->plan_state_->target_exprs_[target_idx];
752 if (
auto const agg_expr = dynamic_cast<const Analyzer::AggExpr*>(target_expr)) {
754 size_t const agg_col_idx =
758 static_cast<int8_t
>(
sizeof(int64_t)));
759 auto const q = agg_expr->get_arg1()->get_constval().doubleval;
761 quantile_params[agg_col_idx] = q;
770 return quantile_params;
775 const int8_t* init_agg_vals_dev_ptr,
778 const unsigned block_size_x,
779 const unsigned grid_size_x) {
782 const auto thread_count = block_size_x * grid_size_x;
783 const auto total_buff_size =
787 std::vector<int8_t*> dev_buffers(thread_count);
789 for (
size_t i = 0; i < thread_count; ++i) {
790 dev_buffers[i] = dev_buffer;
795 dev_ptr, dev_buffers.data(), thread_count *
sizeof(int8_t*));
800 thread_count *
sizeof(int64_t));
803 reinterpret_cast<int8_t*>(dev_buffer + thread_count *
sizeof(int64_t)),
805 thread_count * n *
sizeof(int64_t));
808 reinterpret_cast<int64_t*>(
810 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
814 query_mem_desc.
getRowSize() /
sizeof(int64_t),
820 return {dev_ptr, dev_buffer};
830 const int8_t* init_agg_vals_dev_ptr,
833 const unsigned block_size_x,
834 const unsigned grid_size_x,
835 const int8_t warp_size,
836 const bool can_sort_on_gpu,
837 const bool output_columnar,
841 if (render_allocator) {
845 CHECK(!output_columnar);
848 query_mem_desc, init_agg_vals_dev_ptr,
n, device_id, block_size_x, grid_size_x);
851 auto dev_group_by_buffers =
866 CHECK(dev_group_by_buffers.varlen_output_buffer);
868 reinterpret_cast<CUdeviceptr>(dev_group_by_buffers.varlen_output_buffer);
870 const size_t varlen_output_buf_bytes =
879 if (render_allocator) {
883 CHECK(!render_allocator);
888 auto group_by_dev_buffer = dev_group_by_buffers.data;
890 int8_t* col_widths_dev_ptr{
nullptr};
891 if (output_columnar) {
892 std::vector<int8_t> compact_col_widths(col_count);
893 for (
size_t idx = 0; idx < col_count; ++idx) {
898 col_widths_dev_ptr, compact_col_widths.data(), col_count *
sizeof(int8_t));
900 const int8_t warp_count =
902 const auto num_group_by_buffers =
904 for (
size_t i = 0; i < num_group_by_buffers; i += step) {
905 if (output_columnar) {
907 reinterpret_cast<int64_t*>(group_by_dev_buffer),
908 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
909 dev_group_by_buffers.entry_count,
920 reinterpret_cast<int64_t*>(group_by_dev_buffer),
921 reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
922 dev_group_by_buffers.entry_count,
925 query_mem_desc.
getRowSize() /
sizeof(int64_t),
931 group_by_dev_buffer += groups_buffer_size;
934 return dev_group_by_buffers;
944 const unsigned block_size_x,
945 const unsigned grid_size_x,
946 const bool zero_initialize_buffers) {
949 size_t total_group_by_buffer_size{0};
952 std::vector<size_t> col_byte_offsets;
953 col_byte_offsets.reserve(num_columns);
955 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
957 size_t group_buffer_size =
num_rows_ * col_width;
958 col_byte_offsets.emplace_back(total_group_by_buffer_size);
959 total_group_by_buffer_size =
963 int8_t* dev_buffers_allocation{
nullptr};
965 CHECK(dev_buffers_allocation);
966 if (zero_initialize_buffers) {
970 auto dev_buffers_mem = dev_buffers_allocation;
971 std::vector<int8_t*> dev_buffers(num_columns);
972 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
973 dev_buffers[col_idx] = dev_buffers_allocation + col_byte_offsets[col_idx];
977 dev_ptrs, dev_buffers.data(), num_columns *
sizeof(
CUdeviceptr));
979 return {dev_ptrs, dev_buffers_mem, (size_t)
num_rows_};
985 const size_t entry_count,
988 const unsigned block_size_x,
989 const unsigned grid_size_x) {
992 int8_t* dev_buffer = gpu_group_by_buffers.
data;
995 const size_t original_entry_count = gpu_group_by_buffers.
entry_count;
996 CHECK_LE(entry_count, original_entry_count);
997 size_t output_device_col_offset{0};
998 size_t output_host_col_offset{0};
1002 auto allocator = std::make_unique<CudaAllocator>(
1005 for (
size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1007 const size_t output_device_col_size = original_entry_count * col_width;
1008 const size_t output_host_col_size = entry_count * col_width;
1009 allocator->copyFromDevice(host_buffer + output_host_col_offset,
1010 dev_buffer + output_device_col_offset,
1011 output_host_col_size);
1012 output_device_col_offset =
1013 align_to_int64(output_device_col_offset + output_device_col_size);
1014 output_host_col_offset =
1025 : executor->blockSize() *
1034 int8_t* projection_buffer,
1035 const size_t projection_count) {
1038 constexpr
size_t row_index_width =
sizeof(int64_t);
1039 size_t buffer_offset1{projection_count * row_index_width};
1041 for (
size_t i = 0; i < query_mem_desc.
getSlotCount(); i++) {
1043 auto column_proj_size =
1046 if (buffer_offset1 + column_proj_size >= buffer_offset2) {
1048 std::memmove(projection_buffer + buffer_offset1,
1049 projection_buffer + buffer_offset2,
1052 std::memcpy(projection_buffer + buffer_offset1,
1053 projection_buffer + buffer_offset2,
1065 const size_t projection_count) {
1066 const auto num_allocated_rows =
1068 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1074 num_allocated_rows);
1078 result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1085 const size_t projection_count,
1086 const int device_id) {
1088 const auto num_allocated_rows =
1092 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1095 gpu_group_by_buffers,
1103 result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1109 const size_t entry_count,
1112 const unsigned block_size_x,
1113 const unsigned grid_size_x,
1114 const int device_id,
1115 const bool prepend_index_buffer)
const {
1116 const auto thread_count = block_size_x * grid_size_x;
1118 size_t total_buff_size{0};
1130 gpu_group_by_buffers.
data,
1135 prepend_index_buffer,
1142 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1160 const unsigned total_thread_count,
1161 const int device_id) {
1164 const size_t buffer_start_idx = query_mem_desc.
hasVarlenOutput() ? 1 : 0;
1166 const auto rows_copy = pick_top_n_rows_from_dev_heaps(
1168 reinterpret_cast<int64_t*>(gpu_group_by_buffers.
data),
GpuGroupByBuffers setupTableFunctionGpuBuffers(const QueryMemoryDescriptor &query_mem_desc, const int device_id, const unsigned block_size_x, const unsigned grid_size_x, const bool zero_initialize_buffers)
std::vector< Analyzer::Expr * > target_exprs
size_t getSlotCount() const
size_t getBufferSizeBytes(const RelAlgExecutionUnit &ra_exe_unit, const unsigned thread_count, const ExecutorDeviceType device_type) const
GpuGroupByBuffers create_dev_group_by_buffers(DeviceAllocator *device_allocator, const std::vector< int64_t * > &group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const ExecutorDispatchMode dispatch_mode, const int64_t num_input_rows, const bool prepend_index_buffer, const bool always_init_group_by_on_host, const bool use_bump_allocator, const bool has_varlen_output, Allocator *insitu_allocator)
RenderAllocator * getRenderAllocator(size_t device_id)
robin_hood::unordered_set< int64_t > CountDistinctSet
bool countDistinctDescriptorsLogicallyEmpty() const
size_t getEntryCount() const
bool useCudaBuffers() const
GpuGroupByBuffers prepareTopNHeapsDevBuffer(const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const size_t n, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
size_t count_distinct_bitmap_mem_bytes_
GpuGroupByBuffers createAndInitializeGroupByBufferGpu(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const int device_id, const ExecutorDispatchMode dispatch_mode, const unsigned block_size_x, const unsigned grid_size_x, const int8_t warp_size, const bool can_sort_on_gpu, const bool output_columnar, RenderAllocator *render_allocator)
void initColumnsPerRow(const QueryMemoryDescriptor &query_mem_desc, int8_t *row_ptr, const std::vector< int64_t > &init_vals, const std::vector< int64_t > &bitmap_sizes, const std::vector< QuantileParam > &quantile_params)
boost::multiprecision::number< boost::multiprecision::cpp_int_backend< 64, 64, boost::multiprecision::signed_magnitude, boost::multiprecision::checked, void >> checked_int64_t
void compact_projection_buffer_for_cpu_columnar(const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count)
DeviceAllocator * device_allocator_
void sort_on_gpu(int64_t *val_buff, int32_t *idx_buff, const uint64_t entry_count, const bool desc, const uint32_t chosen_bytes, ThrustAllocator &alloc, const int device_id)
const std::optional< bool > union_all
Streaming Top N algorithm.
TargetInfo get_target_info(const PointerType target_expr, const bool bigint_count)
size_t get_rows_offset_of_heaps(const size_t n, const size_t thread_count)
bool hasVarlenOutput() const
std::vector< int64_t > allocateCountDistinctBuffers(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
unsigned long long CUdeviceptr
int8_t * allocate(const size_t num_bytes, const size_t thread_idx=0) override
std::vector< InputDescriptor > input_descs
bool hasKeylessHash() const
void init_columnar_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t agg_col_count, const int8_t *col_sizes, const bool need_padding, const bool keyless, const int8_t key_size, const size_t block_size_x, const size_t grid_size_x)
CUdeviceptr count_distinct_bitmap_mem_
void check_total_bitmap_memory(const QueryMemoryDescriptor &query_mem_desc)
virtual int8_t * alloc(const size_t num_bytes)=0
size_t getEffectiveKeyWidth() const
int8_t * initColumnarBuffer(T *buffer_ptr, const T init_val, const uint32_t entry_count)
size_t computeNumberOfBuffers(const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const Executor *executor) const
std::vector< QuantileParam > allocateTDigests(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
size_t getRowSize() const
varlen_output_buffer_host_ptr_(nullptr)
void init_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad, const bool keyless, const int8_t warp_size, const size_t block_size_x, const size_t grid_size_x)
const SlotSize & getSlotInfo(const size_t slot_idx) const
std::vector< Analyzer::Expr * > target_exprs_union
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
void compactProjectionBuffersGpu(const QueryMemoryDescriptor &query_mem_desc, Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const size_t projection_count, const int device_id)
bool blocksShareMemory() const
virtual void copyToDevice(void *device_dst, const void *host_src, const size_t num_bytes) const =0
std::vector< int64_t > init_agg_vals_
bool useStreamingTopN() const
size_t getGroupbyColCount() const
void applyStreamingTopNOffsetCpu(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
void fill_empty_key(void *key_ptr, const size_t key_count, const size_t key_width)
virtual void zeroDeviceMem(int8_t *device_ptr, const size_t num_bytes) const =0
bool lazyInitGroups(const ExecutorDeviceType) const
int64_t g_bitmap_memory_limit
size_t g_max_memory_allocation_size
size_t getAllocatedSize() const
bool is_distinct_target(const TargetInfo &target_info)
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
int8_t * count_distinct_bitmap_crt_ptr_
const std::shared_ptr< Analyzer::Estimator > estimator
QueryMemoryInitializer(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int device_id, const ExecutorDeviceType device_type, const ExecutorDispatchMode dispatch_mode, const bool output_columnar, const bool sort_on_gpu, const int outer_table_id, const int64_t num_rows, const std::vector< std::vector< const int8_t * >> &col_buffers, const std::vector< std::vector< uint64_t >> &frag_offsets, RenderAllocatorMap *render_allocator_map, RenderInfo *render_info, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, DeviceAllocator *gpu_allocator, const size_t thread_idx, const Executor *executor)
void initGroupByBuffer(int64_t *buffer, const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const bool output_columnar, const Executor *executor)
int8_t * varlen_output_buffer_host_ptr_
size_t getCountDistinctDescriptorsSize() const
int64_t allocateCountDistinctSet()
QueryDescriptionType getQueryDescriptionType() const
void compactProjectionBuffersCpu(const QueryMemoryDescriptor &query_mem_desc, const size_t projection_count)
std::vector< int64_t * > group_by_buffers_
void initColumnarGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const Executor *executor)
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
void copyGroupByBuffersFromGpu(DeviceAllocator &device_allocator, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit *ra_exe_unit, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer) const
std::optional< size_t > varlenOutputBufferElemSize() const
std::shared_ptr< VarlenOutputInfo > getVarlenOutputInfo()
count_distinct_bitmap_mem_(0)
size_t bitmapPaddedSizeBytes() const
std::vector< int8_t > get_rows_copy_from_heaps(const int64_t *heaps, const size_t heaps_size, const size_t n, const size_t thread_count)
size_t getNextColOffInBytesRowOnly(const int8_t *col_ptr, const size_t col_idx) const
Abstract class for managing device memory allocations.
std::vector< int64_t > get_consistent_frags_sizes(const std::vector< std::vector< uint64_t >> &frag_offsets)
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
CUstream getQueryEngineCudaStreamForDevice(int device_num)
count_distinct_bitmap_host_mem_(nullptr)
size_t get_heap_size(const size_t row_size, const size_t n, const size_t thread_count)
device_allocator_(device_allocator)
bool interleavedBins(const ExecutorDeviceType) const
const ColSlotContext & getColSlotContext() const
bool threadsShareMemory() const
void copyFromTableFunctionGpuBuffers(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
void applyStreamingTopNOffsetGpu(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit &ra_exe_unit, const unsigned total_thread_count, const int device_id)
const auto getGroupByBuffersSize() const
std::vector< TargetInfo > target_exprs_to_infos(const std::vector< Analyzer::Expr * > &targets, const QueryMemoryDescriptor &query_mem_desc)
Basic constructors and methods of the row set interface.
void copy_projection_buffer_from_gpu_columnar(Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count, const int device_id)
bool g_optimize_row_initialization
count_distinct_bitmap_crt_ptr_(nullptr)
std::shared_ptr< VarlenOutputInfo > varlen_output_info_
int64_t get_consistent_frag_size(const std::vector< uint64_t > &frag_offsets)
int8_t * count_distinct_bitmap_host_mem_
count_distinct_bitmap_mem_bytes_(0)
CUdeviceptr varlen_output_buffer_
std::vector< int64_t > init_agg_val_vec(const std::vector< TargetInfo > &targets, const QueryMemoryDescriptor &query_mem_desc)
const int8_t getSlotIndexForSingleSlotCol(const size_t col_idx) const
size_t getBufferColSlotCount() const
const int8_t getLogicalSlotWidthBytes(const size_t slot_idx) const
size_t getColOffInBytes(const size_t col_idx) const
int64_t * alloc_group_by_buffer(const size_t numBytes, RenderAllocatorMap *render_allocator_map, const size_t thread_idx, RowSetMemoryOwner *mem_owner)
void copy_group_by_buffers_from_gpu(DeviceAllocator &device_allocator, const std::vector< int64_t * > &group_by_buffers, const size_t groups_buffer_size, const int8_t *group_by_dev_buffers_mem, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer, const bool has_varlen_output)
std::vector< std::unique_ptr< ResultSet > > result_sets_
virtual void setDeviceMem(int8_t *device_ptr, unsigned char uc, const size_t num_bytes) const =0
void allocateCountDistinctGpuMem(const QueryMemoryDescriptor &query_mem_desc)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
int64_t allocateCountDistinctBitmap(const size_t bitmap_byte_sz)
void initRowGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const int32_t groups_buffer_entry_count, const size_t warp_size, const Executor *executor)
int get_input_idx(RelAlgExecutionUnit const &ra_exe_unit, int const outer_table_id)
std::vector< std::vector< int64_t > > get_col_frag_offsets(const std::vector< Analyzer::Expr * > &target_exprs, const std::vector< std::vector< uint64_t >> &table_frag_offsets)