OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
QueryMemoryInitializer.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
21 #include "GpuMemUtils.h"
23 #include "ResultSet.h"
24 
25 #include "ThirdParty/robin_hood/robin_hood.h"
26 
27 #include <memory>
28 
29 #ifdef HAVE_CUDA
30 #include <cuda.h>
31 #else
32 #include <Shared/nocuda.h>
33 #endif
34 
36  public:
37  // Row-based execution constructor
40  const int device_id,
41  const ExecutorDeviceType device_type,
42  const ExecutorDispatchMode dispatch_mode,
43  const bool output_columnar,
44  const bool sort_on_gpu,
45  const shared::TableKey& outer_table_key,
46  const int64_t num_rows,
47  const std::vector<std::vector<const int8_t*>>& col_buffers,
48  const std::vector<std::vector<uint64_t>>& frag_offsets,
49  RenderAllocatorMap* render_allocator_map,
50  RenderInfo* render_info,
51  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
52  DeviceAllocator* gpu_allocator,
53  const size_t thread_idx,
54  const Executor* executor);
55 
56  // Table functions execution constructor
58  const QueryMemoryDescriptor& query_mem_desc,
59  const int device_id,
60  const ExecutorDeviceType device_type,
61  const int64_t num_rows,
62  const std::vector<std::vector<const int8_t*>>& col_buffers,
63  const std::vector<std::vector<uint64_t>>& frag_offsets,
64  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
65  DeviceAllocator* device_allocator,
66  const Executor* executor);
67 
69 
71 
72  const auto getCountDistinctBitmapBytes() const {
74  }
75 
76  // TODO: lazy init (maybe lazy init count distinct above, too?)
78 
79  const auto getVarlenOutputPtr() const { return varlen_output_buffer_; }
80 
81  ResultSet* getResultSet(const size_t index) const {
82  CHECK_LT(index, result_sets_.size());
83  return result_sets_[index].get();
84  }
85 
86  std::unique_ptr<ResultSet> getResultSetOwned(const size_t index) {
87  CHECK_LT(index, result_sets_.size());
88  return std::move(result_sets_[index]);
89  }
90 
91  void resetResultSet(const size_t index) {
92  CHECK_LT(index, result_sets_.size());
93  result_sets_[index].reset();
94  }
95 
96  int64_t getAggInitValForIndex(const size_t index) const {
97  CHECK_LT(index, init_agg_vals_.size());
98  return init_agg_vals_[index];
99  }
100 
101  const auto getGroupByBuffersPtr() {
102  return reinterpret_cast<int64_t**>(group_by_buffers_.data());
103  }
104 
105  const auto getGroupByBuffersSize() const { return group_by_buffers_.size(); }
106 
107  const auto getNumBuffers() const {
109  return num_buffers_;
110  }
111 
113  const QueryMemoryDescriptor& query_mem_desc,
114  const int device_id,
115  const unsigned block_size_x,
116  const unsigned grid_size_x,
117  const bool zero_initialize_buffers);
118 
120  const QueryMemoryDescriptor& query_mem_desc,
121  const size_t entry_count,
122  const GpuGroupByBuffers& gpu_group_by_buffers,
123  const int device_id,
124  const unsigned block_size_x,
125  const unsigned grid_size_x);
126 
127  void copyGroupByBuffersFromGpu(DeviceAllocator& device_allocator,
128  const QueryMemoryDescriptor& query_mem_desc,
129  const size_t entry_count,
130  const GpuGroupByBuffers& gpu_group_by_buffers,
131  const RelAlgExecutionUnit* ra_exe_unit,
132  const unsigned block_size_x,
133  const unsigned grid_size_x,
134  const int device_id,
135  const bool prepend_index_buffer) const;
136 
137  private:
138  void initGroupByBuffer(int64_t* buffer,
139  const RelAlgExecutionUnit& ra_exe_unit,
140  const QueryMemoryDescriptor& query_mem_desc,
141  const ExecutorDeviceType device_type,
142  const bool output_columnar,
143  const Executor* executor);
144 
145  void initRowGroups(const QueryMemoryDescriptor& query_mem_desc,
146  int64_t* groups_buffer,
147  const std::vector<int64_t>& init_vals,
148  const int32_t groups_buffer_entry_count,
149  const size_t warp_size,
150  const Executor* executor);
151 
152  void initColumnarGroups(const QueryMemoryDescriptor& query_mem_desc,
153  int64_t* groups_buffer,
154  const std::vector<int64_t>& init_vals,
155  const Executor* executor);
156 
157  using ModeIndexSet = robin_hood::unordered_set<size_t>;
158  using QuantileParam = std::optional<double>;
159  void initColumnsPerRow(const QueryMemoryDescriptor& query_mem_desc,
160  int8_t* row_ptr,
161  const std::vector<int64_t>& init_vals,
162  const std::vector<int64_t>& bitmap_sizes,
163  const ModeIndexSet& mode_index_set,
164  const std::vector<QuantileParam>& quantile_params);
165 
166  void allocateCountDistinctGpuMem(const QueryMemoryDescriptor& query_mem_desc);
167 
168  std::vector<int64_t> allocateCountDistinctBuffers(
169  const QueryMemoryDescriptor& query_mem_desc,
170  const bool deferred,
171  const Executor* executor);
172 
173  int64_t allocateCountDistinctBitmap(const size_t bitmap_byte_sz);
174 
175  int64_t allocateCountDistinctSet();
176 
177  ModeIndexSet allocateModes(const QueryMemoryDescriptor& query_mem_desc,
178  const bool deferred,
179  const Executor* executor);
180 
181  std::vector<QuantileParam> allocateTDigests(const QueryMemoryDescriptor& query_mem_desc,
182  const bool deferred,
183  const Executor* executor);
184 
186  const int8_t* init_agg_vals_dev_ptr,
187  const size_t n,
188  const int device_id,
189  const unsigned block_size_x,
190  const unsigned grid_size_x);
191 
193  const RelAlgExecutionUnit& ra_exe_unit,
194  const QueryMemoryDescriptor& query_mem_desc,
195  const int8_t* init_agg_vals_dev_ptr,
196  const int device_id,
197  const ExecutorDispatchMode dispatch_mode,
198  const unsigned block_size_x,
199  const unsigned grid_size_x,
200  const int8_t warp_size,
201  const bool can_sort_on_gpu,
202  const bool output_columnar,
203  RenderAllocator* render_allocator);
204 
205  size_t computeNumberOfBuffers(const QueryMemoryDescriptor& query_mem_desc,
206  const ExecutorDeviceType device_type,
207  const Executor* executor) const;
208 
209  void compactProjectionBuffersCpu(const QueryMemoryDescriptor& query_mem_desc,
210  const size_t projection_count);
211  void compactProjectionBuffersGpu(const QueryMemoryDescriptor& query_mem_desc,
212  Data_Namespace::DataMgr* data_mgr,
213  const GpuGroupByBuffers& gpu_group_by_buffers,
214  const size_t projection_count,
215  const int device_id);
216 
217  void applyStreamingTopNOffsetCpu(const QueryMemoryDescriptor& query_mem_desc,
218  const RelAlgExecutionUnit& ra_exe_unit);
219 
221  const QueryMemoryDescriptor& query_mem_desc,
222  const GpuGroupByBuffers& gpu_group_by_buffers,
223  const RelAlgExecutionUnit& ra_exe_unit,
224  const unsigned total_thread_count,
225  const int device_id);
226 
227  std::shared_ptr<VarlenOutputInfo> getVarlenOutputInfo();
228 
229  const int64_t num_rows_;
230  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner_;
231  std::vector<std::unique_ptr<ResultSet>> result_sets_;
232 
233  std::vector<int64_t> init_agg_vals_;
234 
235  size_t num_buffers_;
236  std::vector<int64_t*> group_by_buffers_;
237  std::shared_ptr<VarlenOutputInfo> varlen_output_info_;
240 
245 
247  std::vector<Data_Namespace::AbstractBuffer*> temporary_buffers_;
248 
249  const size_t thread_idx_;
250 
251  friend class Executor; // Accesses result_sets_
252  friend class QueryExecutionContext;
253 };
GpuGroupByBuffers setupTableFunctionGpuBuffers(const QueryMemoryDescriptor &query_mem_desc, const int device_id, const unsigned block_size_x, const unsigned grid_size_x, const bool zero_initialize_buffers)
#define CHECK_EQ(x, y)
Definition: Logger.h:301
ModeIndexSet allocateModes(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
GpuGroupByBuffers prepareTopNHeapsDevBuffer(const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const size_t n, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
void resetResultSet(const size_t index)
void initColumnsPerRow(const QueryMemoryDescriptor &query_mem_desc, int8_t *row_ptr, const std::vector< int64_t > &init_vals, const std::vector< int64_t > &bitmap_sizes, const ModeIndexSet &mode_index_set, const std::vector< QuantileParam > &quantile_params)
GpuGroupByBuffers createAndInitializeGroupByBufferGpu(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int8_t *init_agg_vals_dev_ptr, const int device_id, const ExecutorDispatchMode dispatch_mode, const unsigned block_size_x, const unsigned grid_size_x, const int8_t warp_size, const bool can_sort_on_gpu, const bool output_columnar, RenderAllocator *render_allocator)
DeviceAllocator * device_allocator_
ExecutorDeviceType
void sort_on_gpu(int64_t *val_buff, int32_t *idx_buff, const uint64_t entry_count, const bool desc, const uint32_t chosen_bytes, ThrustAllocator &alloc, const int device_id)
QueryMemoryInitializer(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const int device_id, const ExecutorDeviceType device_type, const ExecutorDispatchMode dispatch_mode, const bool output_columnar, const bool sort_on_gpu, const shared::TableKey &outer_table_key, const int64_t num_rows, const std::vector< std::vector< const int8_t * >> &col_buffers, const std::vector< std::vector< uint64_t >> &frag_offsets, RenderAllocatorMap *render_allocator_map, RenderInfo *render_info, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, DeviceAllocator *gpu_allocator, const size_t thread_idx, const Executor *executor)
std::vector< int64_t > allocateCountDistinctBuffers(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
unsigned long long CUdeviceptr
Definition: nocuda.h:28
std::optional< double > QuantileParam
size_t computeNumberOfBuffers(const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const Executor *executor) const
std::vector< QuantileParam > allocateTDigests(const QueryMemoryDescriptor &query_mem_desc, const bool deferred, const Executor *executor)
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
ExecutorDispatchMode
void compactProjectionBuffersGpu(const QueryMemoryDescriptor &query_mem_desc, Data_Namespace::DataMgr *data_mgr, const GpuGroupByBuffers &gpu_group_by_buffers, const size_t projection_count, const int device_id)
std::vector< int64_t > init_agg_vals_
std::unique_ptr< ResultSet > getResultSetOwned(const size_t index)
void applyStreamingTopNOffsetCpu(const QueryMemoryDescriptor &query_mem_desc, const RelAlgExecutionUnit &ra_exe_unit)
int64_t getAggInitValForIndex(const size_t index) const
const auto getCountDistinctBitmapPtr() const
void initGroupByBuffer(int64_t *buffer, const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, const bool output_columnar, const Executor *executor)
void compactProjectionBuffersCpu(const QueryMemoryDescriptor &query_mem_desc, const size_t projection_count)
std::vector< int64_t * > group_by_buffers_
void initColumnarGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const Executor *executor)
void copyGroupByBuffersFromGpu(DeviceAllocator &device_allocator, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit *ra_exe_unit, const unsigned block_size_x, const unsigned grid_size_x, const int device_id, const bool prepend_index_buffer) const
#define CHECK_LT(x, y)
Definition: Logger.h:303
std::shared_ptr< VarlenOutputInfo > getVarlenOutputInfo()
const auto getNumBuffers() const
robin_hood::unordered_set< size_t > ModeIndexSet
Abstract class for managing device memory allocations.
const auto getCountDistinctBitmapBytes() const
Descriptor for the result set buffer layout.
const auto getVarlenOutputPtr() const
std::vector< Data_Namespace::AbstractBuffer * > temporary_buffers_
void copyFromTableFunctionGpuBuffers(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const size_t entry_count, const GpuGroupByBuffers &gpu_group_by_buffers, const int device_id, const unsigned block_size_x, const unsigned grid_size_x)
void applyStreamingTopNOffsetGpu(Data_Namespace::DataMgr *data_mgr, const QueryMemoryDescriptor &query_mem_desc, const GpuGroupByBuffers &gpu_group_by_buffers, const RelAlgExecutionUnit &ra_exe_unit, const unsigned total_thread_count, const int device_id)
const auto getGroupByBuffersSize() const
Basic constructors and methods of the row set interface.
std::shared_ptr< VarlenOutputInfo > varlen_output_info_
constexpr double n
Definition: Utm.h:38
ResultSet * getResultSet(const size_t index) const
std::vector< std::unique_ptr< ResultSet > > result_sets_
void allocateCountDistinctGpuMem(const QueryMemoryDescriptor &query_mem_desc)
const auto getVarlenOutputHostPtr() const
int64_t allocateCountDistinctBitmap(const size_t bitmap_byte_sz)
const auto getCountDistinctHostPtr() const
void initRowGroups(const QueryMemoryDescriptor &query_mem_desc, int64_t *groups_buffer, const std::vector< int64_t > &init_vals, const int32_t groups_buffer_entry_count, const size_t warp_size, const Executor *executor)