OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ColSlotContext.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "ColSlotContext.h"
24 
25 #include "../BufferCompaction.h"
26 
27 #include <Analyzer/Analyzer.h>
28 #include <Shared/SqlTypesLayout.h>
29 
30 #include <numeric>
31 #include <stdexcept>
32 
33 extern bool g_bigint_count;
34 
35 ColSlotContext::ColSlotContext(const std::vector<Analyzer::Expr*>& col_expr_list,
36  const std::vector<int64_t>& col_exprs_to_not_project) {
37  // Note that non-projected col exprs could be projected cols that we can lazy fetch or
38  // grouped cols with keyless hash
39  if (!col_exprs_to_not_project.empty()) {
40  CHECK_EQ(col_expr_list.size(), col_exprs_to_not_project.size());
41  }
42  size_t col_expr_idx = 0;
43  col_to_slot_map_.resize(col_expr_list.size());
44  for (const auto col_expr : col_expr_list) {
45  if (!col_exprs_to_not_project.empty() &&
46  col_exprs_to_not_project[col_expr_idx] != -1) {
47  addSlotForColumn(0, 0, col_expr_idx);
48  ++col_expr_idx;
49  continue;
50  }
51  if (!col_expr) {
52  // row index
53  addSlotForColumn(sizeof(int64_t), col_expr_idx);
54  } else {
55  const auto agg_info = get_target_info(col_expr, g_bigint_count);
56  const auto chosen_type = get_compact_type(agg_info);
57 
58  if ((chosen_type.is_string() && chosen_type.get_compression() == kENCODING_NONE) ||
59  chosen_type.is_array()) {
60  addSlotForColumn(sizeof(int64_t), col_expr_idx);
61  addSlotForColumn(sizeof(int64_t), col_expr_idx);
62  ++col_expr_idx;
63  continue;
64  }
65  if (chosen_type.is_geometry()) {
66  if (dynamic_cast<const Analyzer::GeoExpr*>(col_expr)) {
67  CHECK_EQ(chosen_type.get_type(), kPOINT);
68  // Pointer/offset into varlen buffer
69  addSlotForColumn(sizeof(int64_t), col_expr_idx);
70  const int64_t arr_size =
71  chosen_type.get_compression() == kENCODING_GEOINT ? 8 : 16;
73  .insert(std::make_pair(slot_sizes_.size() - 1, arr_size))
74  .second);
75  } else {
76  for (auto i = 0; i < chosen_type.get_physical_coord_cols(); ++i) {
77  addSlotForColumn(sizeof(int64_t), col_expr_idx);
78  addSlotForColumn(sizeof(int64_t), col_expr_idx);
79  }
80  }
81  ++col_expr_idx;
82  continue;
83  }
84 
85  const auto col_expr_bitwidth = get_bit_width(chosen_type);
86 
87  CHECK_EQ(size_t(0), col_expr_bitwidth % 8);
88  addSlotForColumn(static_cast<int8_t>(col_expr_bitwidth >> 3), col_expr_idx);
89  // for average, we'll need to keep the count as well
90  if (agg_info.agg_kind == kAVG) {
91  CHECK(agg_info.is_agg);
92  addSlotForColumn(sizeof(int64_t), col_expr_idx);
93  }
94  }
95  ++col_expr_idx;
96  }
97 }
98 
99 void ColSlotContext::setAllSlotsSize(const int8_t slot_width_size) {
100  const SlotSize ss{slot_width_size, slot_width_size};
101  std::fill(slot_sizes_.begin(), slot_sizes_.end(), ss);
102 }
103 
104 void ColSlotContext::setAllSlotsPaddedSize(const int8_t padded_size) {
105  for (auto& slot_size : slot_sizes_) {
106  slot_size.padded_size = padded_size;
107  }
108 }
109 
110 void ColSlotContext::setAllUnsetSlotsPaddedSize(const int8_t padded_size) {
111  for (auto& slot_size : slot_sizes_) {
112  if (slot_size.padded_size < 0) {
113  slot_size.padded_size = padded_size;
114  }
115  }
116 }
117 
119  for (auto& slot_size : slot_sizes_) {
120  slot_size.padded_size = slot_size.logical_size;
121  }
122 }
123 
125  for (const auto& slot_size : slot_sizes_) {
126  CHECK_GE(slot_size.logical_size, 0);
127  CHECK_LE(slot_size.logical_size, slot_size.padded_size);
128  }
129 }
130 
132  return col_to_slot_map_.size();
133 }
135  return slot_sizes_.size();
136 }
137 
139  return std::accumulate(slot_sizes_.cbegin(),
140  slot_sizes_.cend(),
141  size_t(0),
142  [](size_t sum, const auto& slot_size) {
143  CHECK_GE(slot_size.padded_size, 0);
144  return sum + static_cast<size_t>(slot_size.padded_size);
145  });
146 }
147 
150 }
151 
152 size_t ColSlotContext::getAlignedPaddedSizeForRange(const size_t end) const {
153  return std::accumulate(slot_sizes_.cbegin(),
154  slot_sizes_.cbegin() + end,
155  size_t(0),
156  [](size_t sum, const auto& slot_size) {
157  CHECK_GE(slot_size.padded_size, 0);
158  const auto chosen_bytes =
159  static_cast<size_t>(slot_size.padded_size);
160  if (chosen_bytes == sizeof(int64_t)) {
161  return align_to_int64(sum) + chosen_bytes;
162  } else {
163  return sum + chosen_bytes;
164  }
165  });
166 }
167 
168 size_t ColSlotContext::getTotalBytesOfColumnarBuffers(const size_t entry_count) const {
169  const auto total_bytes = std::accumulate(
170  slot_sizes_.cbegin(),
171  slot_sizes_.cend(),
172  size_t(0),
173  [entry_count](size_t sum, const auto& slot_size) {
174  CHECK_GE(slot_size.padded_size, 0);
175  return sum +
176  align_to_int64(static_cast<size_t>(slot_size.padded_size) * entry_count);
177  });
178  return align_to_int64(total_bytes);
179 }
180 
181 int8_t ColSlotContext::getMinPaddedByteSize(const int8_t actual_min_byte_width) const {
182  if (slot_sizes_.empty()) {
183  return actual_min_byte_width;
184  }
185  const auto min_padded_size = std::min_element(
186  slot_sizes_.cbegin(), slot_sizes_.cend(), [](const auto& lhs, const auto& rhs) {
187  return lhs.padded_size < rhs.padded_size;
188  });
189  return std::min(min_padded_size->padded_size, actual_min_byte_width);
190 }
191 
193  if (slot_sizes_.empty()) {
194  return 8;
195  }
196  size_t compact_width{0};
197  for (const auto& slot_size : slot_sizes_) {
198  if (slot_size.padded_size != 0) {
199  compact_width = slot_size.padded_size;
200  break;
201  }
202  }
203  if (!compact_width) {
204  return 0;
205  }
206  CHECK_GT(compact_width, size_t(0));
207  for (const auto& slot_size : slot_sizes_) {
208  if (slot_size.padded_size == 0) {
209  continue;
210  }
211  CHECK_EQ(static_cast<size_t>(slot_size.padded_size), compact_width);
212  }
213  return compact_width;
214 }
215 
216 size_t ColSlotContext::getColOnlyOffInBytes(const size_t slot_idx) const {
217  CHECK_LT(slot_idx, slot_sizes_.size());
218  auto offset_bytes = getAlignedPaddedSizeForRange(slot_idx);
219  if (slot_sizes_[slot_idx].padded_size == sizeof(int64_t)) {
220  offset_bytes = align_to_int64(offset_bytes);
221  }
222  return offset_bytes;
223 }
224 
226  return slot_sizes_.empty();
227 }
228 
230  slot_sizes_.clear();
231  col_to_slot_map_.clear();
232 }
233 
235  size_t total_bytes{0};
236  for (size_t slot_idx = 0; slot_idx < slot_sizes_.size(); slot_idx++) {
237  auto chosen_bytes = slot_sizes_[slot_idx].padded_size;
238  if (chosen_bytes == sizeof(int64_t)) {
239  const auto aligned_total_bytes = align_to_int64(total_bytes);
240  CHECK_GE(aligned_total_bytes, total_bytes);
241  if (slot_idx >= 1) {
242  const auto padding = aligned_total_bytes - total_bytes;
243  CHECK(padding == 0 || padding == 4);
244  slot_sizes_[slot_idx - 1].padded_size += padding;
245  }
246  total_bytes = aligned_total_bytes;
247  }
248  total_bytes += chosen_bytes;
249  }
250  if (!sort_on_gpu) {
251  const auto aligned_total_bytes = align_to_int64(total_bytes);
252  CHECK_GE(aligned_total_bytes, total_bytes);
253  const auto padding = aligned_total_bytes - total_bytes;
254  CHECK(padding == 0 || padding == 4);
255  slot_sizes_.back().padded_size += padding;
256  }
257 }
258 
260  const std::vector<std::tuple<int8_t, int8_t>>& slots_for_col) {
261  const auto col_idx = col_to_slot_map_.size();
262  col_to_slot_map_.emplace_back();
263  for (const auto& slot_info : slots_for_col) {
264  addSlotForColumn(std::get<1>(slot_info), std::get<0>(slot_info), col_idx);
265  }
266 }
267 
268 void ColSlotContext::addColumnFlatBuffer(const int64_t flatbuffer_size) {
269  const auto col_idx = col_to_slot_map_.size();
270  col_to_slot_map_.emplace_back();
271  addSlotForColumn(0, 0, col_idx);
272  // reusing varlenOutput infrastructure for storing the size of a flatbuffer:
273  varlen_output_slot_map_.insert(std::make_pair(col_idx, flatbuffer_size));
274 }
275 
276 int64_t ColSlotContext::getFlatBufferSize(const size_t slot_idx) const {
277  const auto varlen_map_it = varlen_output_slot_map_.find(slot_idx);
278  if (varlen_map_it == varlen_output_slot_map_.end()) {
279  throw std::runtime_error("Failed to find FlatBuffer map entry for slot " +
280  std::to_string(slot_idx));
281  }
282  return varlen_map_it->second;
283 }
284 
285 bool ColSlotContext::checkSlotUsesFlatBufferFormat(const size_t slot_idx) const {
286  const auto varlen_map_it = varlen_output_slot_map_.find(slot_idx);
287  if (varlen_map_it != varlen_output_slot_map_.end() &&
288  slot_idx < col_to_slot_map_.size() && col_to_slot_map_[slot_idx].size() == 1) {
289  const auto& slot_size = slot_sizes_[col_to_slot_map_[slot_idx][0]];
290  return slot_size.padded_size == 0 &&
291  slot_size.logical_size == 0; // as per addColumnFlatBuffer
292  }
293  return false;
294 }
295 
296 void ColSlotContext::addSlotForColumn(const int8_t logical_size,
297  const size_t column_idx) {
298  addSlotForColumn(-1, logical_size, column_idx);
299 }
300 
301 void ColSlotContext::addSlotForColumn(const int8_t padded_size,
302  const int8_t logical_size,
303  const size_t column_idx) {
304  CHECK_LT(column_idx, col_to_slot_map_.size());
305  col_to_slot_map_[column_idx].push_back(slot_sizes_.size());
306  slot_sizes_.emplace_back(SlotSize{padded_size, logical_size});
307 }
308 
309 int64_t ColSlotContext::varlenOutputElementSize(const size_t slot_idx) const {
310  const auto varlen_map_it = varlen_output_slot_map_.find(slot_idx);
311  if (varlen_map_it == varlen_output_slot_map_.end()) {
312  throw std::runtime_error("Failed to find varlen map entry for slot " +
313  std::to_string(slot_idx));
314  }
315  return varlen_map_it->second;
316 }
Defines data structures for the semantic analysis phase of query processing.
int8_t getMinPaddedByteSize(const int8_t actual_min_byte_width) const
#define CHECK_EQ(x, y)
Definition: Logger.h:301
void alignPaddedSlots(const bool sort_on_gpu)
void sort_on_gpu(int64_t *val_buff, int32_t *idx_buff, const uint64_t entry_count, const bool desc, const uint32_t chosen_bytes, ThrustAllocator &alloc, const int device_id)
std::unordered_map< SlotIndex, ArraySize > varlen_output_slot_map_
#define CHECK_GE(x, y)
Definition: Logger.h:306
size_t getAllSlotsPaddedSize() const
size_t getAllSlotsAlignedPaddedSize() const
std::vector< std::vector< size_t > > col_to_slot_map_
std::vector< SlotSize > slot_sizes_
#define CHECK_GT(x, y)
Definition: Logger.h:305
void setAllSlotsSize(const int8_t slot_width_size)
TargetInfo get_target_info(const Analyzer::Expr *target_expr, const bool bigint_count)
Definition: TargetInfo.h:92
std::string to_string(char const *&&v)
size_t getColOnlyOffInBytes(const size_t slot_idx) const
const SQLTypeInfo get_compact_type(const TargetInfo &target)
size_t get_bit_width(const SQLTypeInfo &ti)
DEVICE void fill(ARGS &&...args)
Definition: gpu_enabled.h:60
void addColumnFlatBuffer(const int64_t flatbuffer_size)
size_t getCompactByteWidth() const
Provides column info and slot info for the output buffer and some metadata helpers.
size_t getAlignedPaddedSizeForRange(const size_t end) const
bool g_bigint_count
DEVICE auto accumulate(ARGS &&...args)
Definition: gpu_enabled.h:42
void validate() const
int64_t varlenOutputElementSize(const size_t slot_idx) const
void addColumn(const std::vector< std::tuple< int8_t, int8_t >> &slots_for_col)
#define CHECK_LT(x, y)
Definition: Logger.h:303
#define CHECK_LE(x, y)
Definition: Logger.h:304
void setAllSlotsPaddedSize(const int8_t padded_size)
bool checkSlotUsesFlatBufferFormat(const size_t slot_idx) const
size_t getTotalBytesOfColumnarBuffers(const size_t entry_count) const
void addSlotForColumn(const int8_t logical_size, const size_t column_idx)
size_t getSlotCount() const
void setAllSlotsPaddedSizeToLogicalSize()
#define CHECK(condition)
Definition: Logger.h:291
size_t getColCount() const
void setAllUnsetSlotsPaddedSize(const int8_t padded_size)
Definition: sqldefs.h:77
int64_t getFlatBufferSize(const size_t slot_idx) const
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)