OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
QueryMemoryInitializer.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "QueryMemoryInitializer.h"
18 
20 #include "Execute.h"
21 #include "GpuInitGroups.h"
22 #include "GpuMemUtils.h"
23 #include "Logger/Logger.h"
26 #include "ResultSet.h"
27 #include "StreamingTopN.h"
28 
29 #include <Shared/checked_alloc.h>
30 
31 // 8 GB, the limit of perfect hash group by under normal conditions
32 int64_t g_bitmap_memory_limit{8LL * 1000 * 1000 * 1000};
33 
34 namespace {
35 
37  const int32_t groups_buffer_entry_count = query_mem_desc.getEntryCount();
38  checked_int64_t total_bytes_per_group = 0;
39  const size_t num_count_distinct_descs =
40  query_mem_desc.getCountDistinctDescriptorsSize();
41  for (size_t i = 0; i < num_count_distinct_descs; i++) {
42  const auto count_distinct_desc = query_mem_desc.getCountDistinctDescriptor(i);
43  if (count_distinct_desc.impl_type_ != CountDistinctImplType::Bitmap) {
44  continue;
45  }
46  total_bytes_per_group += count_distinct_desc.bitmapPaddedSizeBytes();
47  }
48  int64_t total_bytes{0};
49  // Using OutOfHostMemory until we can verify that SlabTooBig would also be properly
50  // caught
51  try {
52  total_bytes = static_cast<int64_t>(total_bytes_per_group * groups_buffer_entry_count);
53  } catch (...) {
54  // Absurd amount of memory, merely computing the number of bits overflows int64_t.
55  // Don't bother to report the real amount, this is unlikely to ever happen.
56  throw OutOfHostMemory(std::numeric_limits<int64_t>::max() / 8);
57  }
58  if (total_bytes >= g_bitmap_memory_limit) {
59  throw OutOfHostMemory(total_bytes);
60  }
61 }
62 
63 int64_t* alloc_group_by_buffer(const size_t numBytes,
64  RenderAllocatorMap* render_allocator_map,
65  const size_t thread_idx,
66  RowSetMemoryOwner* mem_owner) {
67  if (render_allocator_map) {
68  // NOTE(adb): If we got here, we are performing an in-situ rendering query and are not
69  // using CUDA buffers. Therefore we need to allocate result set storage using CPU
70  // memory.
71  const auto gpu_idx = 0; // Only 1 GPU supported in CUDA-disabled rendering mode
72  auto render_allocator_ptr = render_allocator_map->getRenderAllocator(gpu_idx);
73  return reinterpret_cast<int64_t*>(render_allocator_ptr->alloc(numBytes));
74  } else {
75  return reinterpret_cast<int64_t*>(mem_owner->allocate(numBytes, thread_idx));
76  }
77 }
78 
79 inline int64_t get_consistent_frag_size(const std::vector<uint64_t>& frag_offsets) {
80  if (frag_offsets.size() < 2) {
81  return int64_t(-1);
82  }
83  const auto frag_size = frag_offsets[1] - frag_offsets[0];
84  for (size_t i = 2; i < frag_offsets.size(); ++i) {
85  const auto curr_size = frag_offsets[i] - frag_offsets[i - 1];
86  if (curr_size != frag_size) {
87  return int64_t(-1);
88  }
89  }
90  return !frag_size ? std::numeric_limits<int64_t>::max()
91  : static_cast<int64_t>(frag_size);
92 }
93 
94 inline std::vector<int64_t> get_consistent_frags_sizes(
95  const std::vector<std::vector<uint64_t>>& frag_offsets) {
96  if (frag_offsets.empty()) {
97  return {};
98  }
99  std::vector<int64_t> frag_sizes;
100  for (size_t tab_idx = 0; tab_idx < frag_offsets[0].size(); ++tab_idx) {
101  std::vector<uint64_t> tab_offs;
102  for (auto& offsets : frag_offsets) {
103  tab_offs.push_back(offsets[tab_idx]);
104  }
105  frag_sizes.push_back(get_consistent_frag_size(tab_offs));
106  }
107  return frag_sizes;
108 }
109 
110 inline std::vector<int64_t> get_consistent_frags_sizes(
111  const std::vector<Analyzer::Expr*>& target_exprs,
112  const std::vector<int64_t>& table_frag_sizes) {
113  std::vector<int64_t> col_frag_sizes;
114  for (auto expr : target_exprs) {
115  if (const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
116  if (col_var->get_rte_idx() < 0) {
117  CHECK_EQ(-1, col_var->get_rte_idx());
118  col_frag_sizes.push_back(int64_t(-1));
119  } else {
120  col_frag_sizes.push_back(table_frag_sizes[col_var->get_rte_idx()]);
121  }
122  } else {
123  col_frag_sizes.push_back(int64_t(-1));
124  }
125  }
126  return col_frag_sizes;
127 }
128 
129 inline std::vector<std::vector<int64_t>> get_col_frag_offsets(
130  const std::vector<Analyzer::Expr*>& target_exprs,
131  const std::vector<std::vector<uint64_t>>& table_frag_offsets) {
132  std::vector<std::vector<int64_t>> col_frag_offsets;
133  for (auto& table_offsets : table_frag_offsets) {
134  std::vector<int64_t> col_offsets;
135  for (auto expr : target_exprs) {
136  if (const auto col_var = dynamic_cast<Analyzer::ColumnVar*>(expr)) {
137  if (col_var->get_rte_idx() < 0) {
138  CHECK_EQ(-1, col_var->get_rte_idx());
139  col_offsets.push_back(int64_t(-1));
140  } else {
141  CHECK_LT(static_cast<size_t>(col_var->get_rte_idx()), table_offsets.size());
142  col_offsets.push_back(
143  static_cast<int64_t>(table_offsets[col_var->get_rte_idx()]));
144  }
145  } else {
146  col_offsets.push_back(int64_t(-1));
147  }
148  }
149  col_frag_offsets.push_back(col_offsets);
150  }
151  return col_frag_offsets;
152 }
153 
154 // Return the RelAlg input index of outer_table_id based on ra_exe_unit.input_descs.
155 // Used by UNION queries to get the target_exprs corresponding to the current subquery.
156 int get_input_idx(RelAlgExecutionUnit const& ra_exe_unit, int const outer_table_id) {
157  auto match_table_id = [=](auto& desc) { return outer_table_id == desc.getTableId(); };
158  auto& input_descs = ra_exe_unit.input_descs;
159  auto itr = std::find_if(input_descs.begin(), input_descs.end(), match_table_id);
160  return itr == input_descs.end() ? 0 : itr->getNestLevel();
161 }
162 
163 } // namespace
164 
165 // Row-based execution constructor
167  const RelAlgExecutionUnit& ra_exe_unit,
169  const int device_id,
170  const ExecutorDeviceType device_type,
171  const ExecutorDispatchMode dispatch_mode,
172  const bool output_columnar,
173  const bool sort_on_gpu,
174  const int outer_table_id,
175  const int64_t num_rows,
176  const std::vector<std::vector<const int8_t*>>& col_buffers,
177  const std::vector<std::vector<uint64_t>>& frag_offsets,
178  RenderAllocatorMap* render_allocator_map,
179  RenderInfo* render_info,
180  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
181  DeviceAllocator* device_allocator,
182  const size_t thread_idx,
183  const Executor* executor)
184  : num_rows_(num_rows)
185  , row_set_mem_owner_(row_set_mem_owner)
186  , init_agg_vals_(executor->plan_state_->init_agg_vals_)
187  , num_buffers_(computeNumberOfBuffers(query_mem_desc, device_type, executor))
194  , device_allocator_(device_allocator)
195  , thread_idx_(thread_idx) {
196  CHECK(!sort_on_gpu || output_columnar);
197 
198  const auto& consistent_frag_sizes = get_consistent_frags_sizes(frag_offsets);
199  if (consistent_frag_sizes.empty()) {
200  // No fragments in the input, no underlying buffers will be needed.
201  return;
202  }
203  if (!ra_exe_unit.use_bump_allocator) {
204  check_total_bitmap_memory(query_mem_desc);
205  }
206  if (device_type == ExecutorDeviceType::GPU) {
207  allocateCountDistinctGpuMem(query_mem_desc);
208  }
209 
210  if (render_allocator_map || !query_mem_desc.isGroupBy()) {
211  allocateCountDistinctBuffers(query_mem_desc, false, executor);
212  allocateTDigests(query_mem_desc, false, executor);
213  if (render_info && render_info->useCudaBuffers()) {
214  return;
215  }
216  }
217 
218  if (ra_exe_unit.estimator) {
219  return;
220  }
221 
222  const auto thread_count = device_type == ExecutorDeviceType::GPU
223  ? executor->blockSize() * executor->gridSize()
224  : 1;
225 
226  size_t group_buffer_size{0};
227  if (ra_exe_unit.use_bump_allocator) {
228  // For kernel per fragment execution, just allocate a buffer equivalent to the size of
229  // the fragment
230  if (dispatch_mode == ExecutorDispatchMode::KernelPerFragment) {
231  group_buffer_size = num_rows * query_mem_desc.getRowSize();
232  } else {
233  // otherwise, allocate a GPU buffer equivalent to the maximum GPU allocation size
234  group_buffer_size = g_max_memory_allocation_size / query_mem_desc.getRowSize();
235  }
236  } else {
237  group_buffer_size =
238  query_mem_desc.getBufferSizeBytes(ra_exe_unit, thread_count, device_type);
239  }
240  CHECK_GE(group_buffer_size, size_t(0));
241 
242  const auto group_buffers_count = !query_mem_desc.isGroupBy() ? 1 : num_buffers_;
243  int64_t* group_by_buffer_template{nullptr};
244  if (!query_mem_desc.lazyInitGroups(device_type) && group_buffers_count > 1) {
245  group_by_buffer_template = reinterpret_cast<int64_t*>(
246  row_set_mem_owner_->allocate(group_buffer_size, thread_idx_));
247  initGroupByBuffer(group_by_buffer_template,
248  ra_exe_unit,
249  query_mem_desc,
250  device_type,
251  output_columnar,
252  executor);
253  }
254 
255  if (query_mem_desc.interleavedBins(device_type)) {
256  CHECK(query_mem_desc.hasKeylessHash());
257  }
258 
259  const auto step = device_type == ExecutorDeviceType::GPU &&
260  query_mem_desc.threadsShareMemory() &&
261  query_mem_desc.isGroupBy()
262  ? executor->blockSize()
263  : size_t(1);
264  const auto index_buffer_qw = device_type == ExecutorDeviceType::GPU && sort_on_gpu &&
265  query_mem_desc.hasKeylessHash()
266  ? query_mem_desc.getEntryCount()
267  : size_t(0);
268  const auto actual_group_buffer_size =
269  group_buffer_size + index_buffer_qw * sizeof(int64_t);
270  CHECK_GE(actual_group_buffer_size, group_buffer_size);
271 
272  if (query_mem_desc.hasVarlenOutput()) {
273  const auto varlen_buffer_elem_size_opt = query_mem_desc.varlenOutputBufferElemSize();
274  CHECK(varlen_buffer_elem_size_opt); // TODO(adb): relax
275  auto varlen_output_buffer = reinterpret_cast<int64_t*>(row_set_mem_owner_->allocate(
276  query_mem_desc.getEntryCount() * varlen_buffer_elem_size_opt.value()));
277  num_buffers_ += 1;
278  group_by_buffers_.push_back(varlen_output_buffer);
279  }
280 
281  for (size_t i = 0; i < group_buffers_count; i += step) {
282  auto group_by_buffer = alloc_group_by_buffer(actual_group_buffer_size,
283  render_allocator_map,
284  thread_idx_,
285  row_set_mem_owner_.get());
286  if (!query_mem_desc.lazyInitGroups(device_type)) {
287  if (group_by_buffer_template) {
288  memcpy(group_by_buffer + index_buffer_qw,
289  group_by_buffer_template,
290  group_buffer_size);
291  } else {
292  initGroupByBuffer(group_by_buffer + index_buffer_qw,
293  ra_exe_unit,
294  query_mem_desc,
295  device_type,
296  output_columnar,
297  executor);
298  }
299  }
300  group_by_buffers_.push_back(group_by_buffer);
301  for (size_t j = 1; j < step; ++j) {
302  group_by_buffers_.push_back(nullptr);
303  }
304  const bool use_target_exprs_union =
305  ra_exe_unit.union_all && get_input_idx(ra_exe_unit, outer_table_id);
306  const auto& target_exprs = use_target_exprs_union ? ra_exe_unit.target_exprs_union
307  : ra_exe_unit.target_exprs;
308  const auto column_frag_offsets = get_col_frag_offsets(target_exprs, frag_offsets);
309  const auto column_frag_sizes =
310  get_consistent_frags_sizes(target_exprs, consistent_frag_sizes);
311 
312  result_sets_.emplace_back(
313  new ResultSet(target_exprs_to_infos(target_exprs, query_mem_desc),
314  executor->getColLazyFetchInfo(target_exprs),
315  col_buffers,
316  column_frag_offsets,
317  column_frag_sizes,
318  device_type,
319  device_id,
322  executor->getCatalog(),
323  executor->blockSize(),
324  executor->gridSize()));
325  result_sets_.back()->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
326  executor->plan_state_->init_agg_vals_,
328  for (size_t j = 1; j < step; ++j) {
329  result_sets_.emplace_back(nullptr);
330  }
331  }
332 }
333 
334 // Table functions execution constructor
336  const TableFunctionExecutionUnit& exe_unit,
338  const int device_id,
339  const ExecutorDeviceType device_type,
340  const int64_t num_rows,
341  const std::vector<std::vector<const int8_t*>>& col_buffers,
342  const std::vector<std::vector<uint64_t>>& frag_offsets,
343  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
344  DeviceAllocator* device_allocator,
345  const Executor* executor)
346  : num_rows_(num_rows)
347  , row_set_mem_owner_(row_set_mem_owner)
348  , init_agg_vals_(init_agg_val_vec(exe_unit.target_exprs, {}, query_mem_desc))
349  , num_buffers_(1)
356  , device_allocator_(device_allocator)
357  , thread_idx_(0) {
358  // Table functions output columnar, basically treat this as a projection
359  const auto& consistent_frag_sizes = get_consistent_frags_sizes(frag_offsets);
360  if (consistent_frag_sizes.empty()) {
361  // No fragments in the input, no underlying buffers will be needed.
362  return;
363  }
364 
365  const size_t num_columns = query_mem_desc.getBufferColSlotCount();
366  size_t total_group_by_buffer_size{0};
367  for (size_t i = 0; i < num_columns; ++i) {
368  const size_t col_width = exe_unit.target_exprs[i]->get_type_info().get_size();
369  const size_t group_buffer_size = num_rows_ * col_width;
370  total_group_by_buffer_size =
371  align_to_int64(total_group_by_buffer_size + group_buffer_size);
372  }
373 
374  CHECK_EQ(num_buffers_, size_t(1));
375  auto group_by_buffer = alloc_group_by_buffer(
376  total_group_by_buffer_size, nullptr, thread_idx_, row_set_mem_owner.get());
377  group_by_buffers_.push_back(group_by_buffer);
378 
379  const auto column_frag_offsets =
380  get_col_frag_offsets(exe_unit.target_exprs, frag_offsets);
381  const auto column_frag_sizes =
382  get_consistent_frags_sizes(exe_unit.target_exprs, consistent_frag_sizes);
383  result_sets_.emplace_back(
384  new ResultSet(target_exprs_to_infos(exe_unit.target_exprs, query_mem_desc),
385  /*col_lazy_fetch_info=*/{},
386  col_buffers,
387  column_frag_offsets,
388  column_frag_sizes,
389  device_type,
390  device_id,
392  row_set_mem_owner_,
393  executor->getCatalog(),
394  executor->blockSize(),
395  executor->gridSize()));
396  result_sets_.back()->allocateStorage(reinterpret_cast<int8_t*>(group_by_buffer),
397  init_agg_vals_);
398 }
399 
401  int64_t* buffer,
402  const RelAlgExecutionUnit& ra_exe_unit,
404  const ExecutorDeviceType device_type,
405  const bool output_columnar,
406  const Executor* executor) {
407  if (output_columnar) {
408  initColumnarGroups(query_mem_desc, buffer, init_agg_vals_, executor);
409  } else {
410  auto rows_ptr = buffer;
411  auto actual_entry_count = query_mem_desc.getEntryCount();
412  const auto thread_count = device_type == ExecutorDeviceType::GPU
413  ? executor->blockSize() * executor->gridSize()
414  : 1;
415  auto warp_size =
416  query_mem_desc.interleavedBins(device_type) ? executor->warpSize() : 1;
417  if (query_mem_desc.useStreamingTopN()) {
418  const auto node_count_size = thread_count * sizeof(int64_t);
419  memset(rows_ptr, 0, node_count_size);
420  const auto n = ra_exe_unit.sort_info.offset + ra_exe_unit.sort_info.limit;
421  const auto rows_offset = streaming_top_n::get_rows_offset_of_heaps(n, thread_count);
422  memset(rows_ptr + thread_count, -1, rows_offset - node_count_size);
423  rows_ptr += rows_offset / sizeof(int64_t);
424  actual_entry_count = n * thread_count;
425  warp_size = 1;
426  }
427  initRowGroups(query_mem_desc,
428  rows_ptr,
430  actual_entry_count,
431  warp_size,
432  executor);
433  }
434 }
435 
437  int64_t* groups_buffer,
438  const std::vector<int64_t>& init_vals,
439  const int32_t groups_buffer_entry_count,
440  const size_t warp_size,
441  const Executor* executor) {
442  const size_t key_count{query_mem_desc.getGroupbyColCount()};
443  const size_t row_size{query_mem_desc.getRowSize()};
444  const size_t col_base_off{query_mem_desc.getColOffInBytes(0)};
445 
446  auto agg_bitmap_size = allocateCountDistinctBuffers(query_mem_desc, true, executor);
447  auto quantile_params = allocateTDigests(query_mem_desc, true, executor);
448  auto buffer_ptr = reinterpret_cast<int8_t*>(groups_buffer);
449 
450  const auto query_mem_desc_fixedup =
452 
453  auto const is_true = [](auto const& x) { return static_cast<bool>(x); };
454  // not COUNT DISTINCT / APPROX_COUNT_DISTINCT / APPROX_QUANTILE
455  // we fallback to default implementation in that cases
456  if (!std::any_of(agg_bitmap_size.begin(), agg_bitmap_size.end(), is_true) &&
457  !std::any_of(quantile_params.begin(), quantile_params.end(), is_true) &&
459  std::vector<int8_t> sample_row(row_size - col_base_off);
460 
461  initColumnsPerRow(query_mem_desc_fixedup,
462  sample_row.data(),
463  init_vals,
464  agg_bitmap_size,
465  quantile_params);
466 
467  if (query_mem_desc.hasKeylessHash()) {
468  CHECK(warp_size >= 1);
469  CHECK(key_count == 1 || warp_size == 1);
470  for (size_t warp_idx = 0; warp_idx < warp_size; ++warp_idx) {
471  for (size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
472  ++bin, buffer_ptr += row_size) {
473  memcpy(buffer_ptr + col_base_off, sample_row.data(), sample_row.size());
474  }
475  }
476  return;
477  }
478 
479  for (size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
480  ++bin, buffer_ptr += row_size) {
481  memcpy(buffer_ptr + col_base_off, sample_row.data(), sample_row.size());
483  buffer_ptr, key_count, query_mem_desc.getEffectiveKeyWidth());
484  }
485  } else {
486  if (query_mem_desc.hasKeylessHash()) {
487  CHECK(warp_size >= 1);
488  CHECK(key_count == 1 || warp_size == 1);
489  for (size_t warp_idx = 0; warp_idx < warp_size; ++warp_idx) {
490  for (size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
491  ++bin, buffer_ptr += row_size) {
492  initColumnsPerRow(query_mem_desc_fixedup,
493  &buffer_ptr[col_base_off],
494  init_vals,
495  agg_bitmap_size,
496  quantile_params);
497  }
498  }
499  return;
500  }
501 
502  for (size_t bin = 0; bin < static_cast<size_t>(groups_buffer_entry_count);
503  ++bin, buffer_ptr += row_size) {
505  buffer_ptr, key_count, query_mem_desc.getEffectiveKeyWidth());
506  initColumnsPerRow(query_mem_desc_fixedup,
507  &buffer_ptr[col_base_off],
508  init_vals,
509  agg_bitmap_size,
510  quantile_params);
511  }
512  }
513 }
514 
515 namespace {
516 
517 template <typename T>
518 int8_t* initColumnarBuffer(T* buffer_ptr, const T init_val, const uint32_t entry_count) {
519  static_assert(sizeof(T) <= sizeof(int64_t), "Unsupported template type");
520  for (uint32_t i = 0; i < entry_count; ++i) {
521  buffer_ptr[i] = init_val;
522  }
523  return reinterpret_cast<int8_t*>(buffer_ptr + entry_count);
524 }
525 
526 } // namespace
527 
530  int64_t* groups_buffer,
531  const std::vector<int64_t>& init_vals,
532  const Executor* executor) {
533  CHECK(groups_buffer);
534 
535  for (const auto target_expr : executor->plan_state_->target_exprs_) {
536  const auto agg_info = get_target_info(target_expr, g_bigint_count);
537  CHECK(!is_distinct_target(agg_info));
538  }
539  const int32_t agg_col_count = query_mem_desc.getSlotCount();
540  auto buffer_ptr = reinterpret_cast<int8_t*>(groups_buffer);
541 
542  const auto groups_buffer_entry_count = query_mem_desc.getEntryCount();
543  if (!query_mem_desc.hasKeylessHash()) {
544  const size_t key_count{query_mem_desc.getGroupbyColCount()};
545  for (size_t i = 0; i < key_count; ++i) {
546  buffer_ptr = initColumnarBuffer<int64_t>(reinterpret_cast<int64_t*>(buffer_ptr),
547  EMPTY_KEY_64,
548  groups_buffer_entry_count);
549  }
550  }
551 
553  // initializing all aggregate columns:
554  int32_t init_val_idx = 0;
555  for (int32_t i = 0; i < agg_col_count; ++i) {
556  if (query_mem_desc.getPaddedSlotWidthBytes(i) > 0) {
557  CHECK_LT(static_cast<size_t>(init_val_idx), init_vals.size());
558  switch (query_mem_desc.getPaddedSlotWidthBytes(i)) {
559  case 1:
560  buffer_ptr = initColumnarBuffer<int8_t>(
561  buffer_ptr, init_vals[init_val_idx++], groups_buffer_entry_count);
562  break;
563  case 2:
564  buffer_ptr =
565  initColumnarBuffer<int16_t>(reinterpret_cast<int16_t*>(buffer_ptr),
566  init_vals[init_val_idx++],
567  groups_buffer_entry_count);
568  break;
569  case 4:
570  buffer_ptr =
571  initColumnarBuffer<int32_t>(reinterpret_cast<int32_t*>(buffer_ptr),
572  init_vals[init_val_idx++],
573  groups_buffer_entry_count);
574  break;
575  case 8:
576  buffer_ptr =
577  initColumnarBuffer<int64_t>(reinterpret_cast<int64_t*>(buffer_ptr),
578  init_vals[init_val_idx++],
579  groups_buffer_entry_count);
580  break;
581  case 0:
582  break;
583  default:
584  CHECK(false);
585  }
586 
587  buffer_ptr = align_to_int64(buffer_ptr);
588  }
589  }
590  }
591 }
592 
595  int8_t* row_ptr,
596  const std::vector<int64_t>& init_vals,
597  const std::vector<int64_t>& bitmap_sizes,
598  const std::vector<QuantileParam>& quantile_params) {
599  int8_t* col_ptr = row_ptr;
600  size_t init_vec_idx = 0;
601  for (size_t col_idx = 0; col_idx < query_mem_desc.getSlotCount();
602  col_ptr += query_mem_desc.getNextColOffInBytesRowOnly(col_ptr, col_idx++)) {
603  const int64_t bm_sz{bitmap_sizes[col_idx]};
604  int64_t init_val{0};
605  if (bm_sz && query_mem_desc.isGroupBy()) {
606  // COUNT DISTINCT / APPROX_COUNT_DISTINCT
607  CHECK_EQ(static_cast<size_t>(query_mem_desc.getPaddedSlotWidthBytes(col_idx)),
608  sizeof(int64_t));
609  init_val =
611  ++init_vec_idx;
612  } else if (query_mem_desc.isGroupBy() && quantile_params[col_idx]) {
613  auto const q = *quantile_params[col_idx];
614  // allocate for APPROX_QUANTILE only when slot is used
615  init_val = reinterpret_cast<int64_t>(row_set_mem_owner_->nullTDigest(q));
616  ++init_vec_idx;
617  } else {
618  if (query_mem_desc.getPaddedSlotWidthBytes(col_idx) > 0) {
619  CHECK_LT(init_vec_idx, init_vals.size());
620  init_val = init_vals[init_vec_idx++];
621  }
622  }
623  switch (query_mem_desc.getPaddedSlotWidthBytes(col_idx)) {
624  case 1:
625  *col_ptr = static_cast<int8_t>(init_val);
626  break;
627  case 2:
628  *reinterpret_cast<int16_t*>(col_ptr) = (int16_t)init_val;
629  break;
630  case 4:
631  *reinterpret_cast<int32_t*>(col_ptr) = (int32_t)init_val;
632  break;
633  case 8:
634  *reinterpret_cast<int64_t*>(col_ptr) = init_val;
635  break;
636  case 0:
637  continue;
638  default:
639  CHECK(false);
640  }
641  }
642 }
643 
646  if (query_mem_desc.countDistinctDescriptorsLogicallyEmpty()) {
647  return;
648  }
650 
651  size_t total_bytes_per_entry{0};
652  const size_t num_count_distinct_descs =
653  query_mem_desc.getCountDistinctDescriptorsSize();
654  for (size_t i = 0; i < num_count_distinct_descs; i++) {
655  const auto count_distinct_desc = query_mem_desc.getCountDistinctDescriptor(i);
656  if (count_distinct_desc.impl_type_ == CountDistinctImplType::Invalid) {
657  continue;
658  }
659  CHECK(count_distinct_desc.impl_type_ == CountDistinctImplType::Bitmap);
660  total_bytes_per_entry += count_distinct_desc.bitmapPaddedSizeBytes();
661  }
662 
664  total_bytes_per_entry * query_mem_desc.getEntryCount();
665  count_distinct_bitmap_mem_ = reinterpret_cast<CUdeviceptr>(
667  device_allocator_->zeroDeviceMem(reinterpret_cast<int8_t*>(count_distinct_bitmap_mem_),
669 
672 }
673 
674 // deferred is true for group by queries; initGroups will allocate a bitmap
675 // for each group slot
678  const bool deferred,
679  const Executor* executor) {
680  const size_t agg_col_count{query_mem_desc.getSlotCount()};
681  std::vector<int64_t> agg_bitmap_size(deferred ? agg_col_count : 0);
682 
683  CHECK_GE(agg_col_count, executor->plan_state_->target_exprs_.size());
684  for (size_t target_idx = 0; target_idx < executor->plan_state_->target_exprs_.size();
685  ++target_idx) {
686  const auto target_expr = executor->plan_state_->target_exprs_[target_idx];
687  const auto agg_info = get_target_info(target_expr, g_bigint_count);
688  if (is_distinct_target(agg_info)) {
689  CHECK(agg_info.is_agg &&
690  (agg_info.agg_kind == kCOUNT || agg_info.agg_kind == kAPPROX_COUNT_DISTINCT));
691  CHECK(!agg_info.sql_type.is_varlen());
692 
693  const size_t agg_col_idx = query_mem_desc.getSlotIndexForSingleSlotCol(target_idx);
694  CHECK_LT(static_cast<size_t>(agg_col_idx), agg_col_count);
695 
696  CHECK_EQ(static_cast<size_t>(query_mem_desc.getLogicalSlotWidthBytes(agg_col_idx)),
697  sizeof(int64_t));
698  const auto& count_distinct_desc =
699  query_mem_desc.getCountDistinctDescriptor(target_idx);
700  CHECK(count_distinct_desc.impl_type_ != CountDistinctImplType::Invalid);
701  if (count_distinct_desc.impl_type_ == CountDistinctImplType::Bitmap) {
702  const auto bitmap_byte_sz = count_distinct_desc.bitmapPaddedSizeBytes();
703  if (deferred) {
704  agg_bitmap_size[agg_col_idx] = bitmap_byte_sz;
705  } else {
706  init_agg_vals_[agg_col_idx] = allocateCountDistinctBitmap(bitmap_byte_sz);
707  }
708  } else {
709  CHECK(count_distinct_desc.impl_type_ == CountDistinctImplType::UnorderedSet);
710  if (deferred) {
711  agg_bitmap_size[agg_col_idx] = -1;
712  } else {
713  init_agg_vals_[agg_col_idx] = allocateCountDistinctSet();
714  }
715  }
716  }
717  }
718 
719  return agg_bitmap_size;
720 }
721 
722 int64_t QueryMemoryInitializer::allocateCountDistinctBitmap(const size_t bitmap_byte_sz) {
726  count_distinct_bitmap_crt_ptr_ += bitmap_byte_sz;
727  row_set_mem_owner_->addCountDistinctBuffer(
728  ptr, bitmap_byte_sz, /*physial_buffer=*/false);
729  return reinterpret_cast<int64_t>(ptr);
730  }
731  return reinterpret_cast<int64_t>(
732  row_set_mem_owner_->allocateCountDistinctBuffer(bitmap_byte_sz, thread_idx_));
733 }
734 
736  auto count_distinct_set = new CountDistinctSet();
737  row_set_mem_owner_->addCountDistinctSet(count_distinct_set);
738  return reinterpret_cast<int64_t>(count_distinct_set);
739 }
740 
741 std::vector<QueryMemoryInitializer::QuantileParam>
743  const bool deferred,
744  const Executor* executor) {
745  size_t const slot_count = query_mem_desc.getSlotCount();
746  size_t const ntargets = executor->plan_state_->target_exprs_.size();
747  CHECK_GE(slot_count, ntargets);
748  std::vector<QuantileParam> quantile_params(deferred ? slot_count : 0);
749 
750  for (size_t target_idx = 0; target_idx < ntargets; ++target_idx) {
751  auto const target_expr = executor->plan_state_->target_exprs_[target_idx];
752  if (auto const agg_expr = dynamic_cast<const Analyzer::AggExpr*>(target_expr)) {
753  if (agg_expr->get_aggtype() == kAPPROX_QUANTILE) {
754  size_t const agg_col_idx =
755  query_mem_desc.getSlotIndexForSingleSlotCol(target_idx);
756  CHECK_LT(agg_col_idx, slot_count);
757  CHECK_EQ(query_mem_desc.getLogicalSlotWidthBytes(agg_col_idx),
758  static_cast<int8_t>(sizeof(int64_t)));
759  auto const q = agg_expr->get_arg1()->get_constval().doubleval;
760  if (deferred) {
761  quantile_params[agg_col_idx] = q;
762  } else {
763  // allocate for APPROX_QUANTILE only when slot is used
764  init_agg_vals_[agg_col_idx] =
765  reinterpret_cast<int64_t>(row_set_mem_owner_->nullTDigest(q));
766  }
767  }
768  }
769  }
770  return quantile_params;
771 }
772 
775  const int8_t* init_agg_vals_dev_ptr,
776  const size_t n,
777  const int device_id,
778  const unsigned block_size_x,
779  const unsigned grid_size_x) {
780 #ifdef HAVE_CUDA
782  const auto thread_count = block_size_x * grid_size_x;
783  const auto total_buff_size =
784  streaming_top_n::get_heap_size(query_mem_desc.getRowSize(), n, thread_count);
785  int8_t* dev_buffer = device_allocator_->alloc(total_buff_size);
786 
787  std::vector<int8_t*> dev_buffers(thread_count);
788 
789  for (size_t i = 0; i < thread_count; ++i) {
790  dev_buffers[i] = dev_buffer;
791  }
792 
793  auto dev_ptr = device_allocator_->alloc(thread_count * sizeof(int8_t*));
795  dev_ptr, dev_buffers.data(), thread_count * sizeof(int8_t*));
796 
798 
799  device_allocator_->zeroDeviceMem(reinterpret_cast<int8_t*>(dev_buffer),
800  thread_count * sizeof(int64_t));
801 
803  reinterpret_cast<int8_t*>(dev_buffer + thread_count * sizeof(int64_t)),
804  (unsigned char)-1,
805  thread_count * n * sizeof(int64_t));
806 
808  reinterpret_cast<int64_t*>(
809  dev_buffer + streaming_top_n::get_rows_offset_of_heaps(n, thread_count)),
810  reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
811  n * thread_count,
812  query_mem_desc.getGroupbyColCount(),
813  query_mem_desc.getEffectiveKeyWidth(),
814  query_mem_desc.getRowSize() / sizeof(int64_t),
815  query_mem_desc.hasKeylessHash(),
816  1,
817  block_size_x,
818  grid_size_x);
819 
820  return {dev_ptr, dev_buffer};
821 #else
822  UNREACHABLE();
823  return {};
824 #endif
825 }
826 
828  const RelAlgExecutionUnit& ra_exe_unit,
830  const int8_t* init_agg_vals_dev_ptr,
831  const int device_id,
832  const ExecutorDispatchMode dispatch_mode,
833  const unsigned block_size_x,
834  const unsigned grid_size_x,
835  const int8_t warp_size,
836  const bool can_sort_on_gpu,
837  const bool output_columnar,
838  RenderAllocator* render_allocator) {
839 #ifdef HAVE_CUDA
840  if (query_mem_desc.useStreamingTopN()) {
841  if (render_allocator) {
843  }
844  const auto n = ra_exe_unit.sort_info.offset + ra_exe_unit.sort_info.limit;
845  CHECK(!output_columnar);
846 
848  query_mem_desc, init_agg_vals_dev_ptr, n, device_id, block_size_x, grid_size_x);
849  }
850 
851  auto dev_group_by_buffers =
854  query_mem_desc,
855  block_size_x,
856  grid_size_x,
857  device_id,
858  dispatch_mode,
859  num_rows_,
860  can_sort_on_gpu,
861  false,
862  ra_exe_unit.use_bump_allocator,
863  query_mem_desc.hasVarlenOutput(),
864  render_allocator);
865  if (query_mem_desc.hasVarlenOutput()) {
866  CHECK(dev_group_by_buffers.varlen_output_buffer);
868  reinterpret_cast<CUdeviceptr>(dev_group_by_buffers.varlen_output_buffer);
869  CHECK(query_mem_desc.varlenOutputBufferElemSize());
870  const size_t varlen_output_buf_bytes =
871  query_mem_desc.getEntryCount() *
872  query_mem_desc.varlenOutputBufferElemSize().value();
874  row_set_mem_owner_->allocate(varlen_output_buf_bytes, thread_idx_);
876  varlen_output_info_->gpu_start_address = static_cast<int64_t>(varlen_output_buffer_);
878  }
879  if (render_allocator) {
880  CHECK_EQ(size_t(0), render_allocator->getAllocatedSize() % 8);
881  }
882  if (query_mem_desc.lazyInitGroups(ExecutorDeviceType::GPU)) {
883  CHECK(!render_allocator);
884 
885  const size_t step{query_mem_desc.threadsShareMemory() ? block_size_x : 1};
886  size_t groups_buffer_size{query_mem_desc.getBufferSizeBytes(
887  ExecutorDeviceType::GPU, dev_group_by_buffers.entry_count)};
888  auto group_by_dev_buffer = dev_group_by_buffers.data;
889  const size_t col_count = query_mem_desc.getSlotCount();
890  int8_t* col_widths_dev_ptr{nullptr};
891  if (output_columnar) {
892  std::vector<int8_t> compact_col_widths(col_count);
893  for (size_t idx = 0; idx < col_count; ++idx) {
894  compact_col_widths[idx] = query_mem_desc.getPaddedSlotWidthBytes(idx);
895  }
896  col_widths_dev_ptr = device_allocator_->alloc(col_count * sizeof(int8_t));
898  col_widths_dev_ptr, compact_col_widths.data(), col_count * sizeof(int8_t));
899  }
900  const int8_t warp_count =
901  query_mem_desc.interleavedBins(ExecutorDeviceType::GPU) ? warp_size : 1;
902  const auto num_group_by_buffers =
903  getGroupByBuffersSize() - (query_mem_desc.hasVarlenOutput() ? 1 : 0);
904  for (size_t i = 0; i < num_group_by_buffers; i += step) {
905  if (output_columnar) {
907  reinterpret_cast<int64_t*>(group_by_dev_buffer),
908  reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
909  dev_group_by_buffers.entry_count,
910  query_mem_desc.getGroupbyColCount(),
911  col_count,
912  col_widths_dev_ptr,
913  /*need_padding = */ true,
914  query_mem_desc.hasKeylessHash(),
915  sizeof(int64_t),
916  block_size_x,
917  grid_size_x);
918  } else {
920  reinterpret_cast<int64_t*>(group_by_dev_buffer),
921  reinterpret_cast<const int64_t*>(init_agg_vals_dev_ptr),
922  dev_group_by_buffers.entry_count,
923  query_mem_desc.getGroupbyColCount(),
924  query_mem_desc.getEffectiveKeyWidth(),
925  query_mem_desc.getRowSize() / sizeof(int64_t),
926  query_mem_desc.hasKeylessHash(),
927  warp_count,
928  block_size_x,
929  grid_size_x);
930  }
931  group_by_dev_buffer += groups_buffer_size;
932  }
933  }
934  return dev_group_by_buffers;
935 #else
936  UNREACHABLE();
937  return {};
938 #endif
939 }
940 
943  const int device_id,
944  const unsigned block_size_x,
945  const unsigned grid_size_x,
946  const bool zero_initialize_buffers) {
947  const size_t num_columns = query_mem_desc.getBufferColSlotCount();
948  CHECK_GT(num_columns, size_t(0));
949  size_t total_group_by_buffer_size{0};
950  const auto col_slot_context = query_mem_desc.getColSlotContext();
951 
952  std::vector<size_t> col_byte_offsets;
953  col_byte_offsets.reserve(num_columns);
954 
955  for (size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
956  const size_t col_width = col_slot_context.getSlotInfo(col_idx).logical_size;
957  size_t group_buffer_size = num_rows_ * col_width;
958  col_byte_offsets.emplace_back(total_group_by_buffer_size);
959  total_group_by_buffer_size =
960  align_to_int64(total_group_by_buffer_size + group_buffer_size);
961  }
962 
963  int8_t* dev_buffers_allocation{nullptr};
964  dev_buffers_allocation = device_allocator_->alloc(total_group_by_buffer_size);
965  CHECK(dev_buffers_allocation);
966  if (zero_initialize_buffers) {
967  device_allocator_->zeroDeviceMem(dev_buffers_allocation, total_group_by_buffer_size);
968  }
969 
970  auto dev_buffers_mem = dev_buffers_allocation;
971  std::vector<int8_t*> dev_buffers(num_columns);
972  for (size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
973  dev_buffers[col_idx] = dev_buffers_allocation + col_byte_offsets[col_idx];
974  }
975  auto dev_ptrs = device_allocator_->alloc(num_columns * sizeof(CUdeviceptr));
977  dev_ptrs, dev_buffers.data(), num_columns * sizeof(CUdeviceptr));
978 
979  return {dev_ptrs, dev_buffers_mem, (size_t)num_rows_};
980 }
981 
983  Data_Namespace::DataMgr* data_mgr,
984  const QueryMemoryDescriptor& query_mem_desc,
985  const size_t entry_count,
986  const GpuGroupByBuffers& gpu_group_by_buffers,
987  const int device_id,
988  const unsigned block_size_x,
989  const unsigned grid_size_x) {
990  const size_t num_columns = query_mem_desc.getBufferColSlotCount();
991 
992  int8_t* dev_buffer = gpu_group_by_buffers.data;
993  int8_t* host_buffer = reinterpret_cast<int8_t*>(group_by_buffers_[0]);
994 
995  const size_t original_entry_count = gpu_group_by_buffers.entry_count;
996  CHECK_LE(entry_count, original_entry_count);
997  size_t output_device_col_offset{0};
998  size_t output_host_col_offset{0};
999 
1000  const auto col_slot_context = query_mem_desc.getColSlotContext();
1001 
1002  auto allocator = std::make_unique<CudaAllocator>(
1003  data_mgr, device_id, getQueryEngineCudaStreamForDevice(device_id));
1004 
1005  for (size_t col_idx = 0; col_idx < num_columns; ++col_idx) {
1006  const size_t col_width = col_slot_context.getSlotInfo(col_idx).logical_size;
1007  const size_t output_device_col_size = original_entry_count * col_width;
1008  const size_t output_host_col_size = entry_count * col_width;
1009  allocator->copyFromDevice(host_buffer + output_host_col_offset,
1010  dev_buffer + output_device_col_offset,
1011  output_host_col_size);
1012  output_device_col_offset =
1013  align_to_int64(output_device_col_offset + output_device_col_size);
1014  output_host_col_offset =
1015  align_to_int64(output_host_col_offset + output_host_col_size);
1016  }
1017 }
1018 
1020  const QueryMemoryDescriptor& query_mem_desc,
1021  const ExecutorDeviceType device_type,
1022  const Executor* executor) const {
1023  return device_type == ExecutorDeviceType::CPU
1024  ? 1
1025  : executor->blockSize() *
1026  (query_mem_desc.blocksShareMemory() ? 1 : executor->gridSize());
1027 }
1028 
1029 namespace {
1030 
1031 // in-place compaction of output buffer
1033  const QueryMemoryDescriptor& query_mem_desc,
1034  int8_t* projection_buffer,
1035  const size_t projection_count) {
1036  // the first column (row indices) remains unchanged.
1037  CHECK(projection_count <= query_mem_desc.getEntryCount());
1038  constexpr size_t row_index_width = sizeof(int64_t);
1039  size_t buffer_offset1{projection_count * row_index_width};
1040  // other columns are actual non-lazy columns for the projection:
1041  for (size_t i = 0; i < query_mem_desc.getSlotCount(); i++) {
1042  if (query_mem_desc.getPaddedSlotWidthBytes(i) > 0) {
1043  auto column_proj_size =
1044  projection_count * query_mem_desc.getPaddedSlotWidthBytes(i);
1045  auto buffer_offset2 = query_mem_desc.getColOffInBytes(i);
1046  if (buffer_offset1 + column_proj_size >= buffer_offset2) {
1047  // overlapping
1048  std::memmove(projection_buffer + buffer_offset1,
1049  projection_buffer + buffer_offset2,
1050  column_proj_size);
1051  } else {
1052  std::memcpy(projection_buffer + buffer_offset1,
1053  projection_buffer + buffer_offset2,
1054  column_proj_size);
1055  }
1056  buffer_offset1 += align_to_int64(column_proj_size);
1057  }
1058  }
1059 }
1060 
1061 } // namespace
1062 
1064  const QueryMemoryDescriptor& query_mem_desc,
1065  const size_t projection_count) {
1066  const auto num_allocated_rows =
1067  std::min(projection_count, query_mem_desc.getEntryCount());
1068  const size_t buffer_start_idx = query_mem_desc.hasVarlenOutput() ? 1 : 0;
1069 
1070  // copy the results from the main buffer into projection_buffer
1072  query_mem_desc,
1073  reinterpret_cast<int8_t*>(group_by_buffers_[buffer_start_idx]),
1074  num_allocated_rows);
1075 
1076  // update the entry count for the result set, and its underlying storage
1077  CHECK(!result_sets_.empty());
1078  result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1079 }
1080 
1082  const QueryMemoryDescriptor& query_mem_desc,
1083  Data_Namespace::DataMgr* data_mgr,
1084  const GpuGroupByBuffers& gpu_group_by_buffers,
1085  const size_t projection_count,
1086  const int device_id) {
1087  // store total number of allocated rows:
1088  const auto num_allocated_rows =
1089  std::min(projection_count, query_mem_desc.getEntryCount());
1090 
1091  // copy the results from the main buffer into projection_buffer
1092  const size_t buffer_start_idx = query_mem_desc.hasVarlenOutput() ? 1 : 0;
1094  data_mgr,
1095  gpu_group_by_buffers,
1096  query_mem_desc,
1097  reinterpret_cast<int8_t*>(group_by_buffers_[buffer_start_idx]),
1098  num_allocated_rows,
1099  device_id);
1100 
1101  // update the entry count for the result set, and its underlying storage
1102  CHECK(!result_sets_.empty());
1103  result_sets_.front()->updateStorageEntryCount(num_allocated_rows);
1104 }
1105 
1107  DeviceAllocator& device_allocator,
1108  const QueryMemoryDescriptor& query_mem_desc,
1109  const size_t entry_count,
1110  const GpuGroupByBuffers& gpu_group_by_buffers,
1111  const RelAlgExecutionUnit* ra_exe_unit,
1112  const unsigned block_size_x,
1113  const unsigned grid_size_x,
1114  const int device_id,
1115  const bool prepend_index_buffer) const {
1116  const auto thread_count = block_size_x * grid_size_x;
1117 
1118  size_t total_buff_size{0};
1119  if (ra_exe_unit && query_mem_desc.useStreamingTopN()) {
1120  const size_t n = ra_exe_unit->sort_info.offset + ra_exe_unit->sort_info.limit;
1121  total_buff_size =
1122  streaming_top_n::get_heap_size(query_mem_desc.getRowSize(), n, thread_count);
1123  } else {
1124  total_buff_size =
1125  query_mem_desc.getBufferSizeBytes(ExecutorDeviceType::GPU, entry_count);
1126  }
1127  copy_group_by_buffers_from_gpu(device_allocator,
1129  total_buff_size,
1130  gpu_group_by_buffers.data,
1131  query_mem_desc,
1132  block_size_x,
1133  grid_size_x,
1134  device_id,
1135  prepend_index_buffer,
1136  query_mem_desc.hasVarlenOutput());
1137 }
1138 
1140  const QueryMemoryDescriptor& query_mem_desc,
1141  const RelAlgExecutionUnit& ra_exe_unit) {
1142  const size_t buffer_start_idx = query_mem_desc.hasVarlenOutput() ? 1 : 0;
1143  CHECK_EQ(group_by_buffers_.size(), buffer_start_idx + 1);
1144 
1145  const auto rows_copy = streaming_top_n::get_rows_copy_from_heaps(
1146  group_by_buffers_[buffer_start_idx],
1147  query_mem_desc.getBufferSizeBytes(ra_exe_unit, 1, ExecutorDeviceType::CPU),
1148  ra_exe_unit.sort_info.offset + ra_exe_unit.sort_info.limit,
1149  1);
1150  CHECK_EQ(rows_copy.size(),
1151  query_mem_desc.getEntryCount() * query_mem_desc.getRowSize());
1152  memcpy(group_by_buffers_[buffer_start_idx], &rows_copy[0], rows_copy.size());
1153 }
1154 
1156  Data_Namespace::DataMgr* data_mgr,
1157  const QueryMemoryDescriptor& query_mem_desc,
1158  const GpuGroupByBuffers& gpu_group_by_buffers,
1159  const RelAlgExecutionUnit& ra_exe_unit,
1160  const unsigned total_thread_count,
1161  const int device_id) {
1162 #ifdef HAVE_CUDA
1164  const size_t buffer_start_idx = query_mem_desc.hasVarlenOutput() ? 1 : 0;
1165 
1166  const auto rows_copy = pick_top_n_rows_from_dev_heaps(
1167  data_mgr,
1168  reinterpret_cast<int64_t*>(gpu_group_by_buffers.data),
1169  ra_exe_unit,
1170  query_mem_desc,
1171  total_thread_count,
1172  device_id);
1173  CHECK_EQ(
1174  rows_copy.size(),
1175  static_cast<size_t>(query_mem_desc.getEntryCount() * query_mem_desc.getRowSize()));
1176  memcpy(group_by_buffers_[buffer_start_idx], &rows_copy[0], rows_copy.size());
1177 #else
1178  UNREACHABLE();
1179 #endif
1180 }
1181 
1182 std::shared_ptr<VarlenOutputInfo> QueryMemoryInitializer::getVarlenOutputInfo() {
1183  if (varlen_output_info_) {
1184  return varlen_output_info_;
1185  }
1186 
1187  // shared_ptr so that both the ResultSet and QMI can hold on to the varlen info object
1188  // and update it as needed
1189  varlen_output_info_ = std::make_shared<VarlenOutputInfo>(VarlenOutputInfo{
1190  static_cast<int64_t>(varlen_output_buffer_), varlen_output_buffer_host_ptr_});
1191  return varlen_output_info_;
1192 }
GpuGroupByBuffers setupTableFunctionGpuBuffers(const QueryMemoryDescriptor &query_mem_desc, const int device_id, const unsigned block_size_x, const unsigned grid_size_x, const bool zero_initialize_buffers)
std::vector< Analyzer::Expr * > target_exprs
#define CHECK_EQ(x, y)
Definition: Logger.h:230
size_t getBufferSizeBytes(const RelAlgExecutionUnit &ra_exe_unit, const unsigned thread_count, const ExecutorDeviceType device_type) const
GpuGroupByBuffers create_dev_group_by_buffers(DeviceAllocator *device_allocator, const std::vector< int64_t * > &group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const ExecutorDispatchMode dispatch_mode, const int64_t num_input_rows, const bool prepend_index_buffer, const bool always_init_group_by_on_host, const bool use_bump_allocator, const bool has_varlen_output, Allocator *insitu_allocator)
Definition: GpuMemUtils.cpp:70
RenderAllocator * getRenderAllocator(size_t device_id)
robin_hood::unordered_set< int64_t > CountDistinctSet
Definition: CountDistinct.h:35
bool countDistinctDescriptorsLogicallyEmpty() const
bool useCudaBuffers() const
Definition: RenderInfo.cpp:73
#define EMPTY_KEY_64
int8_t logical_size
GpuGroupByBuffers prepareTopNHeapsDevBuffer(const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const size_t n, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
GpuGroupByBuffers createAndInitializeGroupByBufferGpu(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const int device_id, const ExecutorDispatchMode dispatch_mode, const unsigned block_size_x, const unsigned grid_size_x, const int8_t warp_size, const bool can_sort_on_gpu, const bool output_columnar, RenderAllocator *render_allocator)
void initColumnsPerRow(const QueryMemoryDescriptor &query_mem_desc, int8_t *row_ptr, const std::vector< int64_t > &init_vals, const std::vector< int64_t > &bitmap_sizes, const std::vector< QuantileParam > &quantile_params)
boost::multiprecision::number< boost::multiprecision::cpp_int_backend< 64, 64, boost::multiprecision::signed_magnitude, boost::multiprecision::checked, void >> checked_int64_t
void compact_projection_buffer_for_cpu_columnar(const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count)
DeviceAllocator * device_allocator_
ExecutorDeviceType
void sort_on_gpu(int64_t *val_buff, int32_t *idx_buff, const uint64_t entry_count, const bool desc, const uint32_t chosen_bytes, ThrustAllocator &alloc, const int device_id)
const std::optional< bool > union_all
Streaming Top N algorithm.
size_t get_rows_offset_of_heaps(const size_t n, const size_t thread_count)
std::vector< int64_t > allocateCountDistinctBuffers(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
unsigned long long CUdeviceptr
Definition: nocuda.h:28
int8_t * allocate(const size_t num_bytes, const size_t thread_idx=0) override
std::vector< InputDescriptor > input_descs
#define UNREACHABLE()
Definition: Logger.h:266
#define CHECK_GE(x, y)
Definition: Logger.h:235
void init_columnar_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t agg_col_count, const int8_t *col_sizes, const bool need_padding, const bool keyless, const int8_t key_size, const size_t block_size_x, const size_t grid_size_x)
varlen_output_buffer_(0)
void check_total_bitmap_memory(const QueryMemoryDescriptor &query_mem_desc)
virtual int8_t * alloc(const size_t num_bytes)=0
size_t getEffectiveKeyWidth() const
num_buffers_(1)
#define CHECK_GT(x, y)
Definition: Logger.h:234
int8_t * initColumnarBuffer(T *buffer_ptr, const T init_val, const uint32_t entry_count)
TargetInfo get_target_info(const Analyzer::Expr *target_expr, const bool bigint_count)
Definition: TargetInfo.h:97
size_t computeNumberOfBuffers(const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const Executor *executor) const
std::vector< QuantileParam > allocateTDigests(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
varlen_output_buffer_host_ptr_(nullptr)
void init_group_by_buffer_on_device(int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad, const bool keyless, const int8_t warp_size, const size_t block_size_x, const size_t grid_size_x)
const SlotSize & getSlotInfo(const size_t slot_idx) const
std::vector< Analyzer::Expr * > target_exprs_union
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
ExecutorDispatchMode
void compactProjectionBuffersGpu(const QueryMemoryDescriptor &query_mem_desc, Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const size_t projection_count, const int device_id)
const size_t limit
virtual void copyToDevice(void *device_dst, const void *host_src, const size_t num_bytes) const =0
std::vector< int64_t > init_agg_vals_
size_t getGroupbyColCount() const
void applyStreamingTopNOffsetCpu(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
void fill_empty_key(void *key_ptr, const size_t key_count, const size_t key_width)
virtual void zeroDeviceMem(int8_t *device_ptr, const size_t num_bytes) const =0
bool lazyInitGroups(const ExecutorDeviceType) const
bool g_bigint_count
int64_t g_bitmap_memory_limit
size_t g_max_memory_allocation_size
Definition: Execute.cpp:116
size_t getAllocatedSize() const
bool is_distinct_target(const TargetInfo &target_info)
Definition: TargetInfo.h:107
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
const std::shared_ptr< Analyzer::Estimator > estimator
QueryMemoryInitializer(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int device_id, const ExecutorDeviceType device_type, const ExecutorDispatchMode dispatch_mode, const bool output_columnar, const bool sort_on_gpu, const int outer_table_id, const int64_t num_rows, const std::vector< std::vector< const int8_t * >> &col_buffers, const std::vector< std::vector< uint64_t >> &frag_offsets, RenderAllocatorMap *render_allocator_map, RenderInfo *render_info, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, DeviceAllocator *gpu_allocator, const size_t thread_idx, const Executor *executor)
void initGroupByBuffer(int64_t *buffer, const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const bool output_columnar, const Executor *executor)
size_t getCountDistinctDescriptorsSize() const
QueryDescriptionType getQueryDescriptionType() const
void compactProjectionBuffersCpu(const QueryMemoryDescriptor &query_mem_desc, const size_t projection_count)
std::vector< int64_t * > group_by_buffers_
void initColumnarGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const Executor *executor)
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
void copyGroupByBuffersFromGpu(DeviceAllocator &device_allocator, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit *ra_exe_unit, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer) const
std::optional< size_t > varlenOutputBufferElemSize() const
#define CHECK_LT(x, y)
Definition: Logger.h:232
std::shared_ptr< VarlenOutputInfo > getVarlenOutputInfo()
count_distinct_bitmap_mem_(0)
#define CHECK_LE(x, y)
Definition: Logger.h:233
std::vector< int8_t > get_rows_copy_from_heaps(const int64_t *heaps, const size_t heaps_size, const size_t n, const size_t thread_count)
size_t getNextColOffInBytesRowOnly(const int8_t *col_ptr, const size_t col_idx) const
Definition: sqldefs.h:77
Abstract class for managing device memory allocations.
std::vector< int64_t > get_consistent_frags_sizes(const std::vector< std::vector< uint64_t >> &frag_offsets)
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
Definition: ResultSet.cpp:756
CUstream getQueryEngineCudaStreamForDevice(int device_num)
Definition: QueryEngine.cpp:7
count_distinct_bitmap_host_mem_(nullptr)
size_t get_heap_size(const size_t row_size, const size_t n, const size_t thread_count)
device_allocator_(device_allocator)
bool interleavedBins(const ExecutorDeviceType) const
const ColSlotContext & getColSlotContext() const
#define CHECK(condition)
Definition: Logger.h:222
void copyFromTableFunctionGpuBuffers(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
void applyStreamingTopNOffsetGpu(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit &ra_exe_unit, const unsigned total_thread_count, const int device_id)
const auto getGroupByBuffersSize() const
std::vector< TargetInfo > target_exprs_to_infos(const std::vector< Analyzer::Expr * > &targets, const QueryMemoryDescriptor &query_mem_desc)
Basic constructors and methods of the row set interface.
void copy_projection_buffer_from_gpu_columnar(Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const QueryMemoryDescriptor &query_mem_desc, int8_t *projection_buffer, const size_t projection_count, const int device_id)
bool g_optimize_row_initialization
Definition: Execute.cpp:101
count_distinct_bitmap_crt_ptr_(nullptr)
std::shared_ptr< VarlenOutputInfo > varlen_output_info_
constexpr double n
Definition: Utm.h:38
int64_t get_consistent_frag_size(const std::vector< uint64_t > &frag_offsets)
const size_t offset
count_distinct_bitmap_mem_bytes_(0)
std::vector< int64_t > init_agg_val_vec(const std::vector< TargetInfo > &targets, const QueryMemoryDescriptor &query_mem_desc)
const int8_t getSlotIndexForSingleSlotCol(const size_t col_idx) const
const int8_t getLogicalSlotWidthBytes(const size_t slot_idx) const
size_t getColOffInBytes(const size_t col_idx) const
int64_t * alloc_group_by_buffer(const size_t numBytes, RenderAllocatorMap *render_allocator_map, const size_t thread_idx, RowSetMemoryOwner *mem_owner)
void copy_group_by_buffers_from_gpu(DeviceAllocator &device_allocator, const std::vector< int64_t * > &group_by_buffers, const size_t groups_buffer_size, const int8_t *group_by_dev_buffers_mem, const QueryMemoryDescriptor &query_mem_desc, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer, const bool has_varlen_output)
std::vector< std::unique_ptr< ResultSet > > result_sets_
virtual void setDeviceMem(int8_t *device_ptr, unsigned char uc, const size_t num_bytes) const =0
void allocateCountDistinctGpuMem(const QueryMemoryDescriptor &query_mem_desc)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
int64_t allocateCountDistinctBitmap(const size_t bitmap_byte_sz)
void initRowGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const int32_t groups_buffer_entry_count, const size_t warp_size, const Executor *executor)
int get_input_idx(RelAlgExecutionUnit const &ra_exe_unit, int const outer_table_id)
std::vector< std::vector< int64_t > > get_col_frag_offsets(const std::vector< Analyzer::Expr * > &target_exprs, const std::vector< std::vector< uint64_t >> &table_frag_offsets)