OmniSciDB  85c2d10cdc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ResultSetReductionJIT.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2021 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "ResultSetReductionJIT.h"
20 
21 #include "CodeGenerator.h"
22 #include "DynamicWatchdog.h"
23 #include "Execute.h"
24 #include "IRCodegenUtils.h"
26 #include "Shared/likely.h"
27 #include "Shared/quantile.h"
28 
29 #include <llvm/Bitcode/BitcodeReader.h>
30 #include <llvm/IR/Function.h>
31 #include <llvm/IR/IRBuilder.h>
32 #include <llvm/IR/Verifier.h>
33 #include <llvm/Support/SourceMgr.h>
34 #include <llvm/Support/raw_os_ostream.h>
35 
36 extern std::unique_ptr<llvm::Module> g_rt_module;
37 
39 
41 
42 namespace {
43 
44 // Error code to be returned when the watchdog timer triggers during the reduction.
45 const int32_t WATCHDOG_ERROR{-1};
46 // Error code to be returned when the interrupt is triggered during the reduction.
47 const int32_t INTERRUPT_ERROR{10};
48 // Use the interpreter, not the JIT, for a number of entries lower than the threshold.
49 const size_t INTERP_THRESHOLD{25};
50 
51 // Load the value stored at 'ptr' interpreted as 'ptr_type'.
52 Value* emit_load(Value* ptr, Type ptr_type, Function* function) {
53  return function->add<Load>(
54  function->add<Cast>(Cast::CastOp::BitCast, ptr, ptr_type, ""),
55  ptr->label() + "_loaded");
56 }
57 
58 // Load the value stored at 'ptr' as a 32-bit signed integer.
59 Value* emit_load_i32(Value* ptr, Function* function) {
60  return emit_load(ptr, Type::Int32Ptr, function);
61 }
62 
63 // Load the value stored at 'ptr' as a 64-bit signed integer.
64 Value* emit_load_i64(Value* ptr, Function* function) {
65  return emit_load(ptr, Type::Int64Ptr, function);
66 }
67 
68 // Read a 32- or 64-bit integer stored at 'ptr' and sign extend to 64-bit.
69 Value* emit_read_int_from_buff(Value* ptr, const int8_t compact_sz, Function* function) {
70  switch (compact_sz) {
71  case 8: {
72  return emit_load_i64(ptr, function);
73  }
74  case 4: {
75  const auto loaded_val = emit_load_i32(ptr, function);
76  return function->add<Cast>(Cast::CastOp::SExt, loaded_val, Type::Int64, "");
77  }
78  default: {
79  LOG(FATAL) << "Invalid byte width: " << compact_sz;
80  return nullptr;
81  }
82  }
83 }
84 
85 // Emit a runtime call to accumulate into the 'val_ptr' byte address the 'other_ptr'
86 // value when the type is specified as not null.
87 void emit_aggregate_one_value(const std::string& agg_kind,
88  Value* val_ptr,
89  Value* other_ptr,
90  const size_t chosen_bytes,
91  const TargetInfo& agg_info,
92  Function* ir_reduce_one_entry) {
93  const auto sql_type = get_compact_type(agg_info);
94  const auto dest_name = agg_kind + "_dest";
95  if (sql_type.is_fp()) {
96  if (chosen_bytes == sizeof(float)) {
97  const auto agg = ir_reduce_one_entry->add<Cast>(
98  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
99  const auto val = emit_load(other_ptr, Type::FloatPtr, ir_reduce_one_entry);
100  ir_reduce_one_entry->add<Call>(
101  "agg_" + agg_kind + "_float", std::vector<const Value*>{agg, val}, "");
102  } else {
103  CHECK_EQ(chosen_bytes, sizeof(double));
104  const auto agg = ir_reduce_one_entry->add<Cast>(
105  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
106  const auto val = emit_load(other_ptr, Type::DoublePtr, ir_reduce_one_entry);
107  ir_reduce_one_entry->add<Call>(
108  "agg_" + agg_kind + "_double", std::vector<const Value*>{agg, val}, "");
109  }
110  } else {
111  if (chosen_bytes == sizeof(int32_t)) {
112  const auto agg = ir_reduce_one_entry->add<Cast>(
113  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
114  const auto val = emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
115  ir_reduce_one_entry->add<Call>(
116  "agg_" + agg_kind + "_int32", std::vector<const Value*>{agg, val}, "");
117  } else {
118  CHECK_EQ(chosen_bytes, sizeof(int64_t));
119  const auto agg = ir_reduce_one_entry->add<Cast>(
120  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
121  const auto val = emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
122  ir_reduce_one_entry->add<Call>(
123  "agg_" + agg_kind, std::vector<const Value*>{agg, val}, "");
124  }
125  }
126 }
127 
128 // Same as above, but support nullable types as well.
129 void emit_aggregate_one_nullable_value(const std::string& agg_kind,
130  Value* val_ptr,
131  Value* other_ptr,
132  const int64_t init_val,
133  const size_t chosen_bytes,
134  const TargetInfo& agg_info,
135  Function* ir_reduce_one_entry) {
136  const auto dest_name = agg_kind + "_dest";
137  if (agg_info.skip_null_val) {
138  const auto sql_type = get_compact_type(agg_info);
139  if (sql_type.is_fp()) {
140  if (chosen_bytes == sizeof(float)) {
141  const auto agg = ir_reduce_one_entry->add<Cast>(
142  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
143  const auto val = emit_load(other_ptr, Type::FloatPtr, ir_reduce_one_entry);
144  const auto init_val_lv = ir_reduce_one_entry->addConstant<ConstantFP>(
145  *reinterpret_cast<const float*>(may_alias_ptr(&init_val)), Type::Float);
146  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_float_skip_val",
147  std::vector<const Value*>{agg, val, init_val_lv},
148  "");
149  } else {
150  CHECK_EQ(chosen_bytes, sizeof(double));
151  const auto agg = ir_reduce_one_entry->add<Cast>(
152  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
153  const auto val = emit_load(other_ptr, Type::DoublePtr, ir_reduce_one_entry);
154  const auto init_val_lv = ir_reduce_one_entry->addConstant<ConstantFP>(
155  *reinterpret_cast<const double*>(may_alias_ptr(&init_val)), Type::Double);
156  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_double_skip_val",
157  std::vector<const Value*>{agg, val, init_val_lv},
158  "");
159  }
160  } else {
161  if (chosen_bytes == sizeof(int32_t)) {
162  const auto agg = ir_reduce_one_entry->add<Cast>(
163  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
164  const auto val = emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
165  const auto init_val_lv =
166  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int32);
167  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_int32_skip_val",
168  std::vector<const Value*>{agg, val, init_val_lv},
169  "");
170  } else {
171  CHECK_EQ(chosen_bytes, sizeof(int64_t));
172  const auto agg = ir_reduce_one_entry->add<Cast>(
173  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
174  const auto val = emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
175  const auto init_val_lv =
176  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64);
177  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_skip_val",
178  std::vector<const Value*>{agg, val, init_val_lv},
179  "");
180  }
181  }
182  } else {
184  agg_kind, val_ptr, other_ptr, chosen_bytes, agg_info, ir_reduce_one_entry);
185  }
186 }
187 
188 // Emit code to accumulate the 'other_ptr' count into the 'val_ptr' destination.
190  Value* other_ptr,
191  const size_t chosen_bytes,
192  Function* ir_reduce_one_entry) {
193  const auto dest_name = "count_dest";
194  if (chosen_bytes == sizeof(int32_t)) {
195  const auto agg = ir_reduce_one_entry->add<Cast>(
196  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
197  const auto val = emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
198  ir_reduce_one_entry->add<Call>(
199  "agg_sum_int32", std::vector<const Value*>{agg, val}, "");
200  } else {
201  CHECK_EQ(chosen_bytes, sizeof(int64_t));
202  const auto agg = ir_reduce_one_entry->add<Cast>(
203  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
204  const auto val = emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
205  ir_reduce_one_entry->add<Call>("agg_sum", std::vector<const Value*>{agg, val}, "");
206  }
207 }
208 
209 // Emit code to load the value stored at the 'other_pi8' as an integer of the given width
210 // 'chosen_bytes' and write it to the 'slot_pi8' destination only if necessary (the
211 // existing value at destination is the initialization value).
213  Value* other_pi8,
214  const int64_t init_val,
215  const size_t chosen_bytes,
216  Function* ir_reduce_one_entry) {
217  const auto func_name = "write_projection_int" + std::to_string(chosen_bytes * 8);
218  if (chosen_bytes == sizeof(int32_t)) {
219  const auto proj_val = emit_load_i32(other_pi8, ir_reduce_one_entry);
220  ir_reduce_one_entry->add<Call>(
221  func_name,
222  std::vector<const Value*>{
223  slot_pi8,
224  proj_val,
225  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64)},
226  "");
227  } else {
228  CHECK_EQ(chosen_bytes, sizeof(int64_t));
229  const auto proj_val = emit_load_i64(other_pi8, ir_reduce_one_entry);
230  ir_reduce_one_entry->add<Call>(
231  func_name,
232  std::vector<const Value*>{
233  slot_pi8,
234  proj_val,
235  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64)},
236  "");
237  }
238 }
239 
240 // Emit code to load the value stored at the 'other_pi8' as an integer of the given width
241 // 'chosen_bytes' and write it to the 'slot_pi8' destination only if necessary (the
242 // existing value at destination is the initialization value).
244  Value* other_pi8,
245  const int64_t init_val,
246  const size_t chosen_bytes,
247  Function* ir_reduce_one_entry) {
248  if (chosen_bytes == sizeof(int32_t)) {
249  const auto func_name = "checked_single_agg_id_int32";
250  const auto proj_val = emit_load_i32(other_pi8, ir_reduce_one_entry);
251  const auto slot_pi32 = ir_reduce_one_entry->add<Cast>(
252  Cast::CastOp::BitCast, slot_pi8, Type::Int32Ptr, "");
253  return ir_reduce_one_entry->add<Call>(
254  func_name,
255  Type::Int32,
256  std::vector<const Value*>{
257  slot_pi32,
258  proj_val,
259  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int32)},
260  "");
261  } else {
262  const auto func_name = "checked_single_agg_id";
263  CHECK_EQ(chosen_bytes, sizeof(int64_t));
264  const auto proj_val = emit_load_i64(other_pi8, ir_reduce_one_entry);
265  const auto slot_pi64 = ir_reduce_one_entry->add<Cast>(
266  Cast::CastOp::BitCast, slot_pi8, Type::Int64Ptr, "");
267 
268  return ir_reduce_one_entry->add<Call>(
269  func_name,
270  Type::Int32,
271  std::vector<const Value*>{
272  slot_pi64,
273  proj_val,
274  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64)},
275  "");
276  }
277 }
278 
279 std::unique_ptr<Function> create_function(
280  const std::string name,
281  const std::vector<Function::NamedArg>& arg_types,
282  const Type ret_type,
283  const bool always_inline) {
284  return std::make_unique<Function>(name, arg_types, ret_type, always_inline);
285 }
286 
287 // Create the declaration for the 'is_empty_entry' function. Use private linkage since
288 // it's a helper only called from the generated code and mark it as always inline.
289 std::unique_ptr<Function> setup_is_empty_entry(ReductionCode* reduction_code) {
290  return create_function(
291  "is_empty_entry", {{"row_ptr", Type::Int8Ptr}}, Type::Int1, /*always_inline=*/true);
292 }
293 
294 // Create the declaration for the 'reduce_one_entry' helper.
295 std::unique_ptr<Function> setup_reduce_one_entry(ReductionCode* reduction_code,
296  const QueryDescriptionType hash_type) {
297  std::string this_ptr_name;
298  std::string that_ptr_name;
299  switch (hash_type) {
301  this_ptr_name = "this_targets_ptr";
302  that_ptr_name = "that_targets_ptr";
303  break;
304  }
307  this_ptr_name = "this_row_ptr";
308  that_ptr_name = "that_row_ptr";
309  break;
310  }
311  default: {
312  LOG(FATAL) << "Unexpected query description type";
313  }
314  }
315  return create_function("reduce_one_entry",
316  {{this_ptr_name, Type::Int8Ptr},
317  {that_ptr_name, Type::Int8Ptr},
318  {"this_qmd", Type::VoidPtr},
319  {"that_qmd", Type::VoidPtr},
320  {"serialized_varlen_buffer_arg", Type::VoidPtr}},
321  Type::Int32,
322  /*always_inline=*/true);
323 }
324 
325 // Create the declaration for the 'reduce_one_entry_idx' helper.
326 std::unique_ptr<Function> setup_reduce_one_entry_idx(ReductionCode* reduction_code) {
327  return create_function("reduce_one_entry_idx",
328  {{"this_buff", Type::Int8Ptr},
329  {"that_buff", Type::Int8Ptr},
330  {"that_entry_idx", Type::Int32},
331  {"that_entry_count", Type::Int32},
332  {"this_qmd_handle", Type::VoidPtr},
333  {"that_qmd_handle", Type::VoidPtr},
334  {"serialized_varlen_buffer", Type::VoidPtr}},
335  Type::Int32,
336  /*always_inline=*/true);
337 }
338 
339 // Create the declaration for the 'reduce_loop' entry point. Use external linkage, this is
340 // the public API of the generated code directly used from result set reduction.
341 std::unique_ptr<Function> setup_reduce_loop(ReductionCode* reduction_code) {
342  return create_function("reduce_loop",
343  {{"this_buff", Type::Int8Ptr},
344  {"that_buff", Type::Int8Ptr},
345  {"start_index", Type::Int32},
346  {"end_index", Type::Int32},
347  {"that_entry_count", Type::Int32},
348  {"this_qmd_handle", Type::VoidPtr},
349  {"that_qmd_handle", Type::VoidPtr},
350  {"serialized_varlen_buffer", Type::VoidPtr}},
351  Type::Int32,
352  /*always_inline=*/false);
353 }
354 
355 llvm::Function* create_llvm_function(const Function* function, CgenState* cgen_state) {
356  AUTOMATIC_IR_METADATA(cgen_state);
357  auto& ctx = cgen_state->context_;
358  std::vector<llvm::Type*> parameter_types;
359  const auto& arg_types = function->arg_types();
360  for (const auto& named_arg : arg_types) {
361  CHECK(named_arg.type != Type::Void);
362  parameter_types.push_back(llvm_type(named_arg.type, ctx));
363  }
364  const auto func_type = llvm::FunctionType::get(
365  llvm_type(function->ret_type(), ctx), parameter_types, false);
366  const auto linkage = function->always_inline() ? llvm::Function::PrivateLinkage
367  : llvm::Function::ExternalLinkage;
368  auto func =
369  llvm::Function::Create(func_type, linkage, function->name(), cgen_state->module_);
370  const auto arg_it = func->arg_begin();
371  for (size_t i = 0; i < arg_types.size(); ++i) {
372  const auto arg = &*(arg_it + i);
373  arg->setName(arg_types[i].name);
374  }
375  if (function->always_inline()) {
377  }
378  return func;
379 }
380 
381 // Setup the reduction function and helpers declarations, create a module and a code
382 // generation state object.
384  ReductionCode reduction_code{};
385  reduction_code.ir_is_empty = setup_is_empty_entry(&reduction_code);
386  reduction_code.ir_reduce_one_entry = setup_reduce_one_entry(&reduction_code, hash_type);
387  reduction_code.ir_reduce_one_entry_idx = setup_reduce_one_entry_idx(&reduction_code);
388  reduction_code.ir_reduce_loop = setup_reduce_loop(&reduction_code);
389  return reduction_code;
390 }
391 
393  return hash_type == QueryDescriptionType::GroupByBaselineHash ||
396 }
397 
398 // Variable length sample fast path (no serialized variable length buffer).
399 void varlen_buffer_sample(int8_t* this_ptr1,
400  int8_t* this_ptr2,
401  const int8_t* that_ptr1,
402  const int8_t* that_ptr2,
403  const int64_t init_val) {
404  const auto rhs_proj_col = *reinterpret_cast<const int64_t*>(that_ptr1);
405  if (rhs_proj_col != init_val) {
406  *reinterpret_cast<int64_t*>(this_ptr1) = rhs_proj_col;
407  }
408  CHECK(this_ptr2 && that_ptr2);
409  *reinterpret_cast<int64_t*>(this_ptr2) = *reinterpret_cast<const int64_t*>(that_ptr2);
410 }
411 
412 } // namespace
413 
415  const void* serialized_varlen_buffer_handle,
416  int8_t* this_ptr1,
417  int8_t* this_ptr2,
418  const int8_t* that_ptr1,
419  const int8_t* that_ptr2,
420  const int64_t init_val,
421  const int64_t length_to_elems) {
422  if (!serialized_varlen_buffer_handle) {
423  varlen_buffer_sample(this_ptr1, this_ptr2, that_ptr1, that_ptr2, init_val);
424  return;
425  }
426  const auto& serialized_varlen_buffer =
427  *reinterpret_cast<const std::vector<std::string>*>(serialized_varlen_buffer_handle);
428  if (!serialized_varlen_buffer.empty()) {
429  const auto rhs_proj_col = *reinterpret_cast<const int64_t*>(that_ptr1);
430  CHECK_LT(static_cast<size_t>(rhs_proj_col), serialized_varlen_buffer.size());
431  const auto& varlen_bytes_str = serialized_varlen_buffer[rhs_proj_col];
432  const auto str_ptr = reinterpret_cast<const int8_t*>(varlen_bytes_str.c_str());
433  *reinterpret_cast<int64_t*>(this_ptr1) = reinterpret_cast<const int64_t>(str_ptr);
434  *reinterpret_cast<int64_t*>(this_ptr2) =
435  static_cast<int64_t>(varlen_bytes_str.size() / length_to_elems);
436  } else {
437  varlen_buffer_sample(this_ptr1, this_ptr2, that_ptr1, that_ptr2, init_val);
438  }
439 }
440 
441 // Wrappers to be called from the generated code, sharing implementation with the rest of
442 // the system.
443 
445  const int64_t new_set_handle,
446  const int64_t old_set_handle,
447  const void* that_qmd_handle,
448  const void* this_qmd_handle,
449  const int64_t target_logical_idx) {
450  const auto that_qmd = reinterpret_cast<const QueryMemoryDescriptor*>(that_qmd_handle);
451  const auto this_qmd = reinterpret_cast<const QueryMemoryDescriptor*>(this_qmd_handle);
452  const auto& new_count_distinct_desc =
453  that_qmd->getCountDistinctDescriptor(target_logical_idx);
454  const auto& old_count_distinct_desc =
455  this_qmd->getCountDistinctDescriptor(target_logical_idx);
456  CHECK(old_count_distinct_desc.impl_type_ != CountDistinctImplType::Invalid);
457  CHECK(old_count_distinct_desc.impl_type_ == new_count_distinct_desc.impl_type_);
459  new_set_handle, old_set_handle, new_count_distinct_desc, old_count_distinct_desc);
460 }
461 
462 extern "C" RUNTIME_EXPORT void approx_median_jit_rt(const int64_t new_set_handle,
463  const int64_t old_set_handle,
464  const void* that_qmd_handle,
465  const void* this_qmd_handle,
466  const int64_t target_logical_idx) {
467  auto* incoming = reinterpret_cast<quantile::TDigest*>(new_set_handle);
468  if (incoming->centroids().capacity()) {
469  auto* accumulator = reinterpret_cast<quantile::TDigest*>(old_set_handle);
470  accumulator->allocate();
471  accumulator->mergeTDigest(*incoming);
472  }
473 }
474 
476  int8_t* groups_buffer,
477  const int8_t* key,
478  const uint32_t key_count,
479  const void* this_qmd_handle,
480  const int8_t* that_buff,
481  const uint32_t that_entry_idx,
482  const uint32_t that_entry_count,
483  const uint32_t row_size_bytes,
484  int64_t** buff_out,
485  uint8_t* empty) {
486  const auto& this_qmd = *reinterpret_cast<const QueryMemoryDescriptor*>(this_qmd_handle);
487  const auto gvi =
488  result_set::get_group_value_reduction(reinterpret_cast<int64_t*>(groups_buffer),
489  this_qmd.getEntryCount(),
490  reinterpret_cast<const int64_t*>(key),
491  key_count,
492  this_qmd.getEffectiveKeyWidth(),
493  this_qmd,
494  reinterpret_cast<const int64_t*>(that_buff),
495  that_entry_idx,
496  that_entry_count,
497  row_size_bytes >> 3);
498  *buff_out = gvi.first;
499  *empty = gvi.second;
500 }
501 
502 extern "C" RUNTIME_EXPORT uint8_t check_watchdog_rt(const size_t sample_seed) {
503  if (UNLIKELY(g_enable_dynamic_watchdog && (sample_seed & 0x3F) == 0 &&
504  dynamic_watchdog())) {
505  return true;
506  }
507  return false;
508 }
509 
510 extern "C" uint8_t check_interrupt_rt(const size_t sample_seed) {
511  // this func is called iff we enable runtime query interrupt
512  if (UNLIKELY((sample_seed & 0xFFFF) == 0 && check_interrupt())) {
513  return true;
514  }
515  return false;
516 }
517 
519  const std::vector<TargetInfo>& targets,
520  const std::vector<int64_t>& target_init_vals)
521  : query_mem_desc_(query_mem_desc)
522  , targets_(targets)
523  , target_init_vals_(target_init_vals) {}
524 
525 // The code generated for a reduction between two result set buffers is structured in
526 // several functions and their IR is stored in the 'ReductionCode' structure. At a high
527 // level, the pseudocode is:
528 //
529 // func is_empty_func(row_ptr):
530 // ...
531 //
532 // func reduce_func_baseline(this_ptr, that_ptr):
533 // if is_empty_func(that_ptr):
534 // return
535 // for each target in the row:
536 // reduce target from that_ptr into this_ptr
537 //
538 // func reduce_func_perfect_hash(this_ptr, that_ptr):
539 // if is_empty_func(that_ptr):
540 // return
541 // for each target in the row:
542 // reduce target from that_ptr into this_ptr
543 //
544 // func reduce_func_idx(this_buff, that_buff, that_entry_index):
545 // that_ptr = that_result_set[that_entry_index]
546 // # Retrieval of 'this_ptr' is different between perfect hash and baseline.
547 // this_ptr = this_result_set[that_entry_index]
548 // or
549 // get_row(key(that_row_ptr), this_result_setBuffer)
550 // reduce_func_[baseline|perfect_hash](this_ptr, that_ptr)
551 //
552 // func reduce_loop(this_buff, that_buff, start_entry_index, end_entry_index):
553 // for that_entry_index in [start_entry_index, end_entry_index):
554 // reduce_func_idx(this_buff, that_buff, that_entry_index)
555 
557  const auto hash_type = query_mem_desc_.getQueryDescriptionType();
559  return {};
560  }
561  auto reduction_code = setup_functions_ir(hash_type);
562  isEmpty(reduction_code);
566  reduceOneEntryNoCollisions(reduction_code);
567  reduceOneEntryNoCollisionsIdx(reduction_code);
568  break;
569  }
571  reduceOneEntryBaseline(reduction_code);
572  reduceOneEntryBaselineIdx(reduction_code);
573  break;
574  }
575  default: {
576  LOG(FATAL) << "Unexpected query description type";
577  }
578  }
579  reduceLoop(reduction_code);
580  // For small result sets, avoid native code generation and use the interpreter instead.
583  return reduction_code;
584  }
585  std::lock_guard<std::mutex> reduction_guard(ReductionCode::s_reduction_mutex);
586  CodeCacheKey key{cacheKey()};
587  const auto compilation_context = s_code_cache.get(key);
588  if (compilation_context) {
589  auto cpu_context =
590  std::dynamic_pointer_cast<CpuCompilationContext>(compilation_context->first);
591  CHECK(cpu_context);
592  return {reinterpret_cast<ReductionCode::FuncPtr>(cpu_context->func()),
593  nullptr,
594  nullptr,
595  nullptr,
596  std::move(reduction_code.ir_is_empty),
597  std::move(reduction_code.ir_reduce_one_entry),
598  std::move(reduction_code.ir_reduce_one_entry_idx),
599  std::move(reduction_code.ir_reduce_loop)};
600  }
601  reduction_code.cgen_state.reset(new CgenState({}, false));
602  auto cgen_state = reduction_code.cgen_state.get();
603  std::unique_ptr<llvm::Module> module = runtime_module_shallow_copy(cgen_state);
604  cgen_state->module_ = module.get();
605  AUTOMATIC_IR_METADATA(cgen_state);
606  auto ir_is_empty = create_llvm_function(reduction_code.ir_is_empty.get(), cgen_state);
607  auto ir_reduce_one_entry =
608  create_llvm_function(reduction_code.ir_reduce_one_entry.get(), cgen_state);
609  auto ir_reduce_one_entry_idx =
610  create_llvm_function(reduction_code.ir_reduce_one_entry_idx.get(), cgen_state);
611  auto ir_reduce_loop =
612  create_llvm_function(reduction_code.ir_reduce_loop.get(), cgen_state);
613  std::unordered_map<const Function*, llvm::Function*> f;
614  f.emplace(reduction_code.ir_is_empty.get(), ir_is_empty);
615  f.emplace(reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry);
616  f.emplace(reduction_code.ir_reduce_one_entry_idx.get(), ir_reduce_one_entry_idx);
617  f.emplace(reduction_code.ir_reduce_loop.get(), ir_reduce_loop);
618  translate_function(reduction_code.ir_is_empty.get(), ir_is_empty, reduction_code, f);
620  reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code, f);
621  translate_function(reduction_code.ir_reduce_one_entry_idx.get(),
622  ir_reduce_one_entry_idx,
623  reduction_code,
624  f);
626  reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code, f);
627  reduction_code.llvm_reduce_loop = ir_reduce_loop;
628  reduction_code.module = std::move(module);
630  return finalizeReductionCode(std::move(reduction_code),
631  ir_is_empty,
632  ir_reduce_one_entry,
633  ir_reduce_one_entry_idx,
634  key);
635 }
636 
638  // Clear stub cache to avoid crash caused by non-deterministic static destructor order
639  // of LLVM context and the cache.
642  g_rt_module = nullptr;
643 }
644 
645 void ResultSetReductionJIT::isEmpty(const ReductionCode& reduction_code) const {
646  auto ir_is_empty = reduction_code.ir_is_empty.get();
649  Value* key{nullptr};
650  Value* empty_key_val{nullptr};
651  const auto keys_ptr = ir_is_empty->arg(0);
656  CHECK_LT(static_cast<size_t>(query_mem_desc_.getTargetIdxForKey()),
657  target_init_vals_.size());
658  const int64_t target_slot_off = result_set::get_byteoff_of_slot(
660  const auto slot_ptr = ir_is_empty->add<GetElementPtr>(
661  keys_ptr,
662  ir_is_empty->addConstant<ConstantInt>(target_slot_off, Type::Int32),
663  "is_empty_slot_ptr");
664  const auto compact_sz =
666  key = emit_read_int_from_buff(slot_ptr, compact_sz, ir_is_empty);
667  empty_key_val = ir_is_empty->addConstant<ConstantInt>(
669  } else {
671  case 4: {
674  key = emit_load_i32(keys_ptr, ir_is_empty);
675  empty_key_val = ir_is_empty->addConstant<ConstantInt>(EMPTY_KEY_32, Type::Int32);
676  break;
677  }
678  case 8: {
679  key = emit_load_i64(keys_ptr, ir_is_empty);
680  empty_key_val = ir_is_empty->addConstant<ConstantInt>(EMPTY_KEY_64, Type::Int64);
681  break;
682  }
683  default:
684  LOG(FATAL) << "Invalid key width";
685  }
686  }
687  const auto ret =
688  ir_is_empty->add<ICmp>(ICmp::Predicate::EQ, key, empty_key_val, "is_key_empty");
689  ir_is_empty->add<Ret>(ret);
690 }
691 
693  const ReductionCode& reduction_code) const {
694  auto ir_reduce_one_entry = reduction_code.ir_reduce_one_entry.get();
695  const auto this_row_ptr = ir_reduce_one_entry->arg(0);
696  const auto that_row_ptr = ir_reduce_one_entry->arg(1);
697  const auto that_is_empty =
698  ir_reduce_one_entry->add<Call>(reduction_code.ir_is_empty.get(),
699  std::vector<const Value*>{that_row_ptr},
700  "that_is_empty");
701  ir_reduce_one_entry->add<ReturnEarly>(
702  that_is_empty, ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32), "");
703 
704  const auto key_bytes = get_key_bytes_rowwise(query_mem_desc_);
705  if (key_bytes) { // copy the key from right hand side
706  ir_reduce_one_entry->add<MemCpy>(
707  this_row_ptr,
708  that_row_ptr,
709  ir_reduce_one_entry->addConstant<ConstantInt>(key_bytes, Type::Int32));
710  }
711 
712  const auto key_bytes_with_padding = align_to_int64(key_bytes);
713  const auto key_bytes_lv =
714  ir_reduce_one_entry->addConstant<ConstantInt>(key_bytes_with_padding, Type::Int32);
715  const auto this_targets_start_ptr = ir_reduce_one_entry->add<GetElementPtr>(
716  this_row_ptr, key_bytes_lv, "this_targets_start");
717  const auto that_targets_start_ptr = ir_reduce_one_entry->add<GetElementPtr>(
718  that_row_ptr, key_bytes_lv, "that_targets_start");
719 
721  ir_reduce_one_entry, this_targets_start_ptr, that_targets_start_ptr);
722 }
723 
725  Function* ir_reduce_one_entry,
726  Value* this_targets_start_ptr,
727  Value* that_targets_start_ptr) const {
728  const auto& col_slot_context = query_mem_desc_.getColSlotContext();
729  Value* this_targets_ptr = this_targets_start_ptr;
730  Value* that_targets_ptr = that_targets_start_ptr;
731  size_t init_agg_val_idx = 0;
732  for (size_t target_logical_idx = 0; target_logical_idx < targets_.size();
733  ++target_logical_idx) {
734  const auto& target_info = targets_[target_logical_idx];
735  const auto& slots_for_col = col_slot_context.getSlotsForCol(target_logical_idx);
736  Value* this_ptr2{nullptr};
737  Value* that_ptr2{nullptr};
738 
739  bool two_slot_target{false};
740  if (target_info.is_agg &&
741  (target_info.agg_kind == kAVG ||
742  (target_info.agg_kind == kSAMPLE && target_info.sql_type.is_varlen()))) {
743  // Note that this assumes if one of the slot pairs in a given target is an array,
744  // all slot pairs are arrays. Currently this is true for all geo targets, but we
745  // should better codify and store this information in the future
746  two_slot_target = true;
747  }
748 
749  for (size_t target_slot_idx = slots_for_col.front();
750  target_slot_idx < slots_for_col.back() + 1;
751  target_slot_idx += 2) {
752  const auto slot_off_val = query_mem_desc_.getPaddedSlotWidthBytes(target_slot_idx);
753  const auto slot_off =
754  ir_reduce_one_entry->addConstant<ConstantInt>(slot_off_val, Type::Int32);
755  if (UNLIKELY(two_slot_target)) {
756  const auto desc = "target_" + std::to_string(target_logical_idx) + "_second_slot";
757  this_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
758  this_targets_ptr, slot_off, "this_" + desc);
759  that_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
760  that_targets_ptr, slot_off, "that_" + desc);
761  }
762  reduceOneSlot(this_targets_ptr,
763  this_ptr2,
764  that_targets_ptr,
765  that_ptr2,
766  target_info,
767  target_logical_idx,
768  target_slot_idx,
769  init_agg_val_idx,
770  slots_for_col.front(),
771  ir_reduce_one_entry);
772  auto increment_agg_val_idx_maybe =
773  [&init_agg_val_idx, &target_logical_idx, this](const int slot_count) {
775  query_mem_desc_.getTargetGroupbyIndex(target_logical_idx) < 0) {
776  init_agg_val_idx += slot_count;
777  }
778  };
779  if (target_logical_idx + 1 == targets_.size() &&
780  target_slot_idx + 1 >= slots_for_col.back()) {
781  break;
782  }
783  const auto next_desc =
784  "target_" + std::to_string(target_logical_idx + 1) + "_first_slot";
785  if (UNLIKELY(two_slot_target)) {
786  increment_agg_val_idx_maybe(2);
787  const auto two_slot_off = ir_reduce_one_entry->addConstant<ConstantInt>(
788  slot_off_val + query_mem_desc_.getPaddedSlotWidthBytes(target_slot_idx + 1),
789  Type::Int32);
790  this_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
791  this_targets_ptr, two_slot_off, "this_" + next_desc);
792  that_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
793  that_targets_ptr, two_slot_off, "that_" + next_desc);
794  } else {
795  increment_agg_val_idx_maybe(1);
796  this_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
797  this_targets_ptr, slot_off, "this_" + next_desc);
798  that_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
799  that_targets_ptr, slot_off, "that_" + next_desc);
800  }
801  }
802  }
803  ir_reduce_one_entry->add<Ret>(
804  ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32));
805 }
806 
808  const ReductionCode& reduction_code) const {
809  auto ir_reduce_one_entry = reduction_code.ir_reduce_one_entry.get();
810  const auto this_targets_ptr_arg = ir_reduce_one_entry->arg(0);
811  const auto that_targets_ptr_arg = ir_reduce_one_entry->arg(1);
812  Value* this_ptr1 = this_targets_ptr_arg;
813  Value* that_ptr1 = that_targets_ptr_arg;
814  size_t j = 0;
815  size_t init_agg_val_idx = 0;
816  for (size_t target_logical_idx = 0; target_logical_idx < targets_.size();
817  ++target_logical_idx) {
818  const auto& target_info = targets_[target_logical_idx];
819  Value* this_ptr2{nullptr};
820  Value* that_ptr2{nullptr};
821  if (target_info.is_agg &&
822  (target_info.agg_kind == kAVG ||
823  (target_info.agg_kind == kSAMPLE && target_info.sql_type.is_varlen()))) {
824  const auto desc = "target_" + std::to_string(target_logical_idx) + "_second_slot";
825  const auto second_slot_rel_off =
826  ir_reduce_one_entry->addConstant<ConstantInt>(sizeof(int64_t), Type::Int32);
827  this_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
828  this_ptr1, second_slot_rel_off, "this_" + desc);
829  that_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
830  that_ptr1, second_slot_rel_off, "that_" + desc);
831  }
832  reduceOneSlot(this_ptr1,
833  this_ptr2,
834  that_ptr1,
835  that_ptr2,
836  target_info,
837  target_logical_idx,
838  j,
839  init_agg_val_idx,
840  j,
841  ir_reduce_one_entry);
842  if (target_logical_idx + 1 == targets_.size()) {
843  break;
844  }
846  init_agg_val_idx = advance_slot(init_agg_val_idx, target_info, false);
847  } else {
848  if (query_mem_desc_.getTargetGroupbyIndex(target_logical_idx) < 0) {
849  init_agg_val_idx = advance_slot(init_agg_val_idx, target_info, false);
850  }
851  }
852  j = advance_slot(j, target_info, false);
853  const auto next_desc =
854  "target_" + std::to_string(target_logical_idx + 1) + "_first_slot";
855  auto next_slot_rel_off = ir_reduce_one_entry->addConstant<ConstantInt>(
856  init_agg_val_idx * sizeof(int64_t), Type::Int32);
857  this_ptr1 = ir_reduce_one_entry->add<GetElementPtr>(
858  this_targets_ptr_arg, next_slot_rel_off, next_desc);
859  that_ptr1 = ir_reduce_one_entry->add<GetElementPtr>(
860  that_targets_ptr_arg, next_slot_rel_off, next_desc);
861  }
862  ir_reduce_one_entry->add<Ret>(
863  ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32));
864 }
865 
867  const ReductionCode& reduction_code) const {
868  auto ir_reduce_one_entry_idx = reduction_code.ir_reduce_one_entry_idx.get();
873  const auto this_buff = ir_reduce_one_entry_idx->arg(0);
874  const auto that_buff = ir_reduce_one_entry_idx->arg(1);
875  const auto entry_idx = ir_reduce_one_entry_idx->arg(2);
876  const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
877  const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
878  const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
879  const auto row_bytes = ir_reduce_one_entry_idx->addConstant<ConstantInt>(
881  const auto entry_idx_64 = ir_reduce_one_entry_idx->add<Cast>(
882  Cast::CastOp::SExt, entry_idx, Type::Int64, "entry_idx_64");
883  const auto row_off_in_bytes = ir_reduce_one_entry_idx->add<BinaryOperator>(
884  BinaryOperator::BinaryOp::Mul, entry_idx_64, row_bytes, "row_off_in_bytes");
885  const auto this_row_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
886  this_buff, row_off_in_bytes, "this_row_ptr");
887  const auto that_row_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
888  that_buff, row_off_in_bytes, "that_row_ptr");
889  const auto reduce_rc = ir_reduce_one_entry_idx->add<Call>(
890  reduction_code.ir_reduce_one_entry.get(),
891  std::vector<const Value*>{this_row_ptr,
892  that_row_ptr,
893  this_qmd_handle,
894  that_qmd_handle,
895  serialized_varlen_buffer_arg},
896  "");
897  ir_reduce_one_entry_idx->add<Ret>(reduce_rc);
898 }
899 
901  const ReductionCode& reduction_code) const {
902  auto ir_reduce_one_entry_idx = reduction_code.ir_reduce_one_entry_idx.get();
907  const auto this_buff = ir_reduce_one_entry_idx->arg(0);
908  const auto that_buff = ir_reduce_one_entry_idx->arg(1);
909  const auto that_entry_idx = ir_reduce_one_entry_idx->arg(2);
910  const auto that_entry_count = ir_reduce_one_entry_idx->arg(3);
911  const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
912  const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
913  const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
914  const auto row_bytes = ir_reduce_one_entry_idx->addConstant<ConstantInt>(
916  const auto that_entry_idx_64 = ir_reduce_one_entry_idx->add<Cast>(
917  Cast::CastOp::SExt, that_entry_idx, Type::Int64, "that_entry_idx_64");
918  const auto that_row_off_in_bytes =
919  ir_reduce_one_entry_idx->add<BinaryOperator>(BinaryOperator::BinaryOp::Mul,
920  that_entry_idx_64,
921  row_bytes,
922  "that_row_off_in_bytes");
923  const auto that_row_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
924  that_buff, that_row_off_in_bytes, "that_row_ptr");
925  const auto that_is_empty =
926  ir_reduce_one_entry_idx->add<Call>(reduction_code.ir_is_empty.get(),
927  std::vector<const Value*>{that_row_ptr},
928  "that_is_empty");
929  ir_reduce_one_entry_idx->add<ReturnEarly>(
930  that_is_empty,
931  ir_reduce_one_entry_idx->addConstant<ConstantInt>(0, Type::Int32),
932  "");
933  const auto key_count = query_mem_desc_.getGroupbyColCount();
934  const auto one_element =
935  ir_reduce_one_entry_idx->addConstant<ConstantInt>(1, Type::Int32);
936  const auto this_targets_ptr_i64_ptr = ir_reduce_one_entry_idx->add<Alloca>(
937  Type::Int64Ptr, one_element, "this_targets_ptr_out");
938  const auto this_is_empty_ptr =
939  ir_reduce_one_entry_idx->add<Alloca>(Type::Int8, one_element, "this_is_empty_out");
940  ir_reduce_one_entry_idx->add<ExternalCall>(
941  "get_group_value_reduction_rt",
942  Type::Void,
943  std::vector<const Value*>{
944  this_buff,
945  that_row_ptr,
946  ir_reduce_one_entry_idx->addConstant<ConstantInt>(key_count, Type::Int32),
947  this_qmd_handle,
948  that_buff,
949  that_entry_idx,
950  that_entry_count,
951  row_bytes,
952  this_targets_ptr_i64_ptr,
953  this_is_empty_ptr},
954  "");
955  const auto this_targets_ptr_i64 = ir_reduce_one_entry_idx->add<Load>(
956  this_targets_ptr_i64_ptr, "this_targets_ptr_i64");
957  auto this_is_empty =
958  ir_reduce_one_entry_idx->add<Load>(this_is_empty_ptr, "this_is_empty");
959  this_is_empty = ir_reduce_one_entry_idx->add<Cast>(
960  Cast::CastOp::Trunc, this_is_empty, Type::Int1, "this_is_empty_bool");
961  ir_reduce_one_entry_idx->add<ReturnEarly>(
962  this_is_empty,
963  ir_reduce_one_entry_idx->addConstant<ConstantInt>(0, Type::Int32),
964  "");
965  const auto key_qw_count = get_slot_off_quad(query_mem_desc_);
966  const auto this_targets_ptr = ir_reduce_one_entry_idx->add<Cast>(
967  Cast::CastOp::BitCast, this_targets_ptr_i64, Type::Int8Ptr, "this_targets_ptr");
968  const auto key_byte_count = key_qw_count * sizeof(int64_t);
969  const auto key_byte_count_lv =
970  ir_reduce_one_entry_idx->addConstant<ConstantInt>(key_byte_count, Type::Int32);
971  const auto that_targets_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
972  that_row_ptr, key_byte_count_lv, "that_targets_ptr");
973  const auto reduce_rc = ir_reduce_one_entry_idx->add<Call>(
974  reduction_code.ir_reduce_one_entry.get(),
975  std::vector<const Value*>{this_targets_ptr,
976  that_targets_ptr,
977  this_qmd_handle,
978  that_qmd_handle,
979  serialized_varlen_buffer_arg},
980  "");
981  ir_reduce_one_entry_idx->add<Ret>(reduce_rc);
982 }
983 
984 namespace {
985 
986 void generate_loop_body(For* for_loop,
987  Function* ir_reduce_loop,
988  Function* ir_reduce_one_entry_idx,
989  Value* this_buff,
990  Value* that_buff,
991  Value* start_index,
992  Value* that_entry_count,
993  Value* this_qmd_handle,
994  Value* that_qmd_handle,
995  Value* serialized_varlen_buffer) {
996  const auto that_entry_idx = for_loop->add<BinaryOperator>(
997  BinaryOperator::BinaryOp::Add, for_loop->iter(), start_index, "that_entry_idx");
998  const auto sample_seed =
999  for_loop->add<Cast>(Cast::CastOp::SExt, that_entry_idx, Type::Int64, "");
1001  const auto checker_rt_name =
1002  g_enable_dynamic_watchdog ? "check_watchdog_rt" : "check_interrupt_rt";
1003  const auto error_code = g_enable_dynamic_watchdog ? WATCHDOG_ERROR : INTERRUPT_ERROR;
1004  const auto checker_triggered = for_loop->add<ExternalCall>(
1005  checker_rt_name, Type::Int8, std::vector<const Value*>{sample_seed}, "");
1006  const auto interrupt_triggered_bool =
1007  for_loop->add<ICmp>(ICmp::Predicate::NE,
1008  checker_triggered,
1009  ir_reduce_loop->addConstant<ConstantInt>(0, Type::Int8),
1010  "");
1011  for_loop->add<ReturnEarly>(
1012  interrupt_triggered_bool,
1013  ir_reduce_loop->addConstant<ConstantInt>(error_code, Type::Int32),
1014  "");
1015  }
1016  const auto reduce_rc =
1017  for_loop->add<Call>(ir_reduce_one_entry_idx,
1018  std::vector<const Value*>{this_buff,
1019  that_buff,
1020  that_entry_idx,
1021  that_entry_count,
1022  this_qmd_handle,
1023  that_qmd_handle,
1024  serialized_varlen_buffer},
1025  "");
1026 
1027  auto reduce_rc_bool =
1028  for_loop->add<ICmp>(ICmp::Predicate::NE,
1029  reduce_rc,
1030  ir_reduce_loop->addConstant<ConstantInt>(0, Type::Int32),
1031  "");
1032  for_loop->add<ReturnEarly>(reduce_rc_bool, reduce_rc, "");
1033 }
1034 
1035 } // namespace
1036 
1037 void ResultSetReductionJIT::reduceLoop(const ReductionCode& reduction_code) const {
1038  auto ir_reduce_loop = reduction_code.ir_reduce_loop.get();
1039  const auto this_buff_arg = ir_reduce_loop->arg(0);
1040  const auto that_buff_arg = ir_reduce_loop->arg(1);
1041  const auto start_index_arg = ir_reduce_loop->arg(2);
1042  const auto end_index_arg = ir_reduce_loop->arg(3);
1043  const auto that_entry_count_arg = ir_reduce_loop->arg(4);
1044  const auto this_qmd_handle_arg = ir_reduce_loop->arg(5);
1045  const auto that_qmd_handle_arg = ir_reduce_loop->arg(6);
1046  const auto serialized_varlen_buffer_arg = ir_reduce_loop->arg(7);
1047  For* for_loop =
1048  static_cast<For*>(ir_reduce_loop->add<For>(start_index_arg, end_index_arg, ""));
1049  generate_loop_body(for_loop,
1050  ir_reduce_loop,
1051  reduction_code.ir_reduce_one_entry_idx.get(),
1052  this_buff_arg,
1053  that_buff_arg,
1054  start_index_arg,
1055  that_entry_count_arg,
1056  this_qmd_handle_arg,
1057  that_qmd_handle_arg,
1058  serialized_varlen_buffer_arg);
1059  ir_reduce_loop->add<Ret>(ir_reduce_loop->addConstant<ConstantInt>(0, Type::Int32));
1060 }
1061 
1063  Value* this_ptr2,
1064  Value* that_ptr1,
1065  Value* that_ptr2,
1066  const TargetInfo& target_info,
1067  const size_t target_logical_idx,
1068  const size_t target_slot_idx,
1069  const size_t init_agg_val_idx,
1070  const size_t first_slot_idx_for_target,
1071  Function* ir_reduce_one_entry) const {
1073  if (query_mem_desc_.getTargetGroupbyIndex(target_logical_idx) >= 0) {
1074  return;
1075  }
1076  }
1077  const bool float_argument_input = takes_float_argument(target_info);
1078  const auto chosen_bytes = result_set::get_width_for_slot(
1079  target_slot_idx, float_argument_input, query_mem_desc_);
1080  CHECK_LT(init_agg_val_idx, target_init_vals_.size());
1081  auto init_val = target_init_vals_[init_agg_val_idx];
1082  if (target_info.is_agg &&
1083  (target_info.agg_kind != kSINGLE_VALUE && target_info.agg_kind != kSAMPLE)) {
1084  reduceOneAggregateSlot(this_ptr1,
1085  this_ptr2,
1086  that_ptr1,
1087  that_ptr2,
1088  target_info,
1089  target_logical_idx,
1090  target_slot_idx,
1091  init_val,
1092  chosen_bytes,
1093  ir_reduce_one_entry);
1094  } else if (target_info.agg_kind == kSINGLE_VALUE) {
1095  const auto checked_rc = emit_checked_write_projection(
1096  this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1097 
1098  auto checked_rc_bool = ir_reduce_one_entry->add<ICmp>(
1100  checked_rc,
1101  ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32),
1102  "");
1103 
1104  ir_reduce_one_entry->add<ReturnEarly>(checked_rc_bool, checked_rc, "");
1105 
1106  } else {
1108  this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1109  if (target_info.agg_kind == kSAMPLE && target_info.sql_type.is_varlen()) {
1110  CHECK(this_ptr2 && that_ptr2);
1111  size_t length_to_elems{0};
1112  if (target_info.sql_type.is_geometry()) {
1113  // TODO: Assumes hard-coded sizes for geometry targets
1114  length_to_elems = target_slot_idx == first_slot_idx_for_target ? 1 : 4;
1115  } else {
1116  const auto& elem_ti = target_info.sql_type.get_elem_type();
1117  length_to_elems = target_info.sql_type.is_string() ? 1 : elem_ti.get_size();
1118  }
1119  const auto serialized_varlen_buffer_arg = ir_reduce_one_entry->arg(4);
1120  ir_reduce_one_entry->add<ExternalCall>(
1121  "serialized_varlen_buffer_sample",
1122  Type::Void,
1123  std::vector<const Value*>{
1124  serialized_varlen_buffer_arg,
1125  this_ptr1,
1126  this_ptr2,
1127  that_ptr1,
1128  that_ptr2,
1129  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64),
1130  ir_reduce_one_entry->addConstant<ConstantInt>(length_to_elems,
1131  Type::Int64)},
1132  "");
1133  }
1134  }
1135 }
1136 
1138  Value* this_ptr2,
1139  Value* that_ptr1,
1140  Value* that_ptr2,
1141  const TargetInfo& target_info,
1142  const size_t target_logical_idx,
1143  const size_t target_slot_idx,
1144  const int64_t init_val,
1145  const int8_t chosen_bytes,
1146  Function* ir_reduce_one_entry) const {
1147  switch (target_info.agg_kind) {
1148  case kCOUNT:
1149  case kAPPROX_COUNT_DISTINCT: {
1150  if (is_distinct_target(target_info)) {
1151  CHECK_EQ(static_cast<size_t>(chosen_bytes), sizeof(int64_t));
1153  this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1154  break;
1155  }
1156  CHECK_EQ(int64_t(0), init_val);
1157  emit_aggregate_one_count(this_ptr1, that_ptr1, chosen_bytes, ir_reduce_one_entry);
1158  break;
1159  }
1160  case kAPPROX_MEDIAN:
1161  CHECK_EQ(chosen_bytes, static_cast<int8_t>(sizeof(int64_t)));
1163  this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1164  break;
1165  case kAVG: {
1166  // Ignore float argument compaction for count component for fear of its overflow
1167  emit_aggregate_one_count(this_ptr2,
1168  that_ptr2,
1169  query_mem_desc_.getPaddedSlotWidthBytes(target_slot_idx),
1170  ir_reduce_one_entry);
1171  }
1172  // fall thru
1173  case kSUM: {
1175  this_ptr1,
1176  that_ptr1,
1177  init_val,
1178  chosen_bytes,
1179  target_info,
1180  ir_reduce_one_entry);
1181  break;
1182  }
1183  case kMIN: {
1185  this_ptr1,
1186  that_ptr1,
1187  init_val,
1188  chosen_bytes,
1189  target_info,
1190  ir_reduce_one_entry);
1191  break;
1192  }
1193  case kMAX: {
1195  this_ptr1,
1196  that_ptr1,
1197  init_val,
1198  chosen_bytes,
1199  target_info,
1200  ir_reduce_one_entry);
1201  break;
1202  }
1203  default:
1204  LOG(FATAL) << "Invalid aggregate type";
1205  }
1206 }
1207 
1209  Value* this_ptr1,
1210  Value* that_ptr1,
1211  const size_t target_logical_idx,
1212  Function* ir_reduce_one_entry) const {
1214  const auto old_set_handle = emit_load_i64(this_ptr1, ir_reduce_one_entry);
1215  const auto new_set_handle = emit_load_i64(that_ptr1, ir_reduce_one_entry);
1216  const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1217  const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1218  ir_reduce_one_entry->add<ExternalCall>(
1219  "count_distinct_set_union_jit_rt",
1220  Type::Void,
1221  std::vector<const Value*>{
1222  new_set_handle,
1223  old_set_handle,
1224  that_qmd_arg,
1225  this_qmd_arg,
1226  ir_reduce_one_entry->addConstant<ConstantInt>(target_logical_idx, Type::Int64)},
1227  "");
1228 }
1229 
1231  Value* this_ptr1,
1232  Value* that_ptr1,
1233  const size_t target_logical_idx,
1234  Function* ir_reduce_one_entry) const {
1236  const auto old_set_handle = emit_load_i64(this_ptr1, ir_reduce_one_entry);
1237  const auto new_set_handle = emit_load_i64(that_ptr1, ir_reduce_one_entry);
1238  const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1239  const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1240  ir_reduce_one_entry->add<ExternalCall>(
1241  "approx_median_jit_rt",
1242  Type::Void,
1243  std::vector<const Value*>{
1244  new_set_handle,
1245  old_set_handle,
1246  that_qmd_arg,
1247  this_qmd_arg,
1248  ir_reduce_one_entry->addConstant<ConstantInt>(target_logical_idx, Type::Int64)},
1249  "");
1250 }
1251 
1253  ReductionCode reduction_code,
1254  const llvm::Function* ir_is_empty,
1255  const llvm::Function* ir_reduce_one_entry,
1256  const llvm::Function* ir_reduce_one_entry_idx,
1257  const CodeCacheKey& key) const {
1258  CompilationOptions co{
1260 
1261 #ifdef NDEBUG
1262  LOG(IR) << "Reduction Loop:\n"
1263  << serialize_llvm_object(reduction_code.llvm_reduce_loop);
1264  LOG(IR) << "Reduction Is Empty Func:\n" << serialize_llvm_object(ir_is_empty);
1265  LOG(IR) << "Reduction One Entry Func:\n" << serialize_llvm_object(ir_reduce_one_entry);
1266  LOG(IR) << "Reduction One Entry Idx Func:\n"
1267  << serialize_llvm_object(ir_reduce_one_entry_idx);
1268 #else
1269  LOG(IR) << serialize_llvm_object(reduction_code.cgen_state->module_);
1270 #endif
1271 
1272  reduction_code.module.release();
1274  reduction_code.llvm_reduce_loop, {reduction_code.llvm_reduce_loop}, co);
1275  reduction_code.func_ptr = reinterpret_cast<ReductionCode::FuncPtr>(
1276  ee->getPointerToFunction(reduction_code.llvm_reduce_loop));
1277 
1278  auto cpu_compilation_context = std::make_shared<CpuCompilationContext>(std::move(ee));
1279  cpu_compilation_context->setFunctionPointer(reduction_code.llvm_reduce_loop);
1280  reduction_code.compilation_context = cpu_compilation_context;
1282  reduction_code.compilation_context,
1283  reduction_code.llvm_reduce_loop->getParent(),
1284  s_code_cache);
1285  return reduction_code;
1286 }
1287 
1288 namespace {
1289 
1290 std::string target_info_key(const TargetInfo& target_info) {
1291  return std::to_string(target_info.is_agg) + "\n" +
1292  std::to_string(target_info.agg_kind) + "\n" +
1293  target_info.sql_type.get_type_name() + "\n" +
1294  std::to_string(target_info.sql_type.get_notnull()) + "\n" +
1295  target_info.agg_arg_type.get_type_name() + "\n" +
1296  std::to_string(target_info.agg_arg_type.get_notnull()) + "\n" +
1297  std::to_string(target_info.skip_null_val) + "\n" +
1298  std::to_string(target_info.is_distinct);
1299 }
1300 
1301 } // namespace
1302 
1303 std::string ResultSetReductionJIT::cacheKey() const {
1304  std::vector<std::string> target_init_vals_strings;
1305  std::transform(target_init_vals_.begin(),
1306  target_init_vals_.end(),
1307  std::back_inserter(target_init_vals_strings),
1308  [](const int64_t v) { return std::to_string(v); });
1309  const auto target_init_vals_key =
1310  boost::algorithm::join(target_init_vals_strings, ", ");
1311  std::vector<std::string> targets_strings;
1312  std::transform(
1313  targets_.begin(),
1314  targets_.end(),
1315  std::back_inserter(targets_strings),
1316  [](const TargetInfo& target_info) { return target_info_key(target_info); });
1317  const auto targets_key = boost::algorithm::join(targets_strings, ", ");
1318  return query_mem_desc_.reductionKey() + "\n" + target_init_vals_key + "\n" +
1319  targets_key;
1320 }
1321 
1323  const auto hash_type = query_mem_desc_.getQueryDescriptionType();
1324  auto reduction_code = setup_functions_ir(hash_type);
1326  isEmpty(reduction_code);
1327  reduceOneEntryNoCollisions(reduction_code);
1328  reduceOneEntryNoCollisionsIdx(reduction_code);
1329  reduceLoop(reduction_code);
1330  reduction_code.cgen_state.reset(new CgenState({}, false));
1331  auto cgen_state = reduction_code.cgen_state.get();
1332  std::unique_ptr<llvm::Module> module(runtime_module_shallow_copy(cgen_state));
1333 
1334  cgen_state->module_ = module.get();
1335  AUTOMATIC_IR_METADATA(cgen_state);
1336  auto ir_is_empty = create_llvm_function(reduction_code.ir_is_empty.get(), cgen_state);
1337  auto ir_reduce_one_entry =
1338  create_llvm_function(reduction_code.ir_reduce_one_entry.get(), cgen_state);
1339  auto ir_reduce_one_entry_idx =
1340  create_llvm_function(reduction_code.ir_reduce_one_entry_idx.get(), cgen_state);
1341  auto ir_reduce_loop =
1342  create_llvm_function(reduction_code.ir_reduce_loop.get(), cgen_state);
1343  std::unordered_map<const Function*, llvm::Function*> f;
1344  f.emplace(reduction_code.ir_is_empty.get(), ir_is_empty);
1345  f.emplace(reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry);
1346  f.emplace(reduction_code.ir_reduce_one_entry_idx.get(), ir_reduce_one_entry_idx);
1347  f.emplace(reduction_code.ir_reduce_loop.get(), ir_reduce_loop);
1348  translate_function(reduction_code.ir_is_empty.get(), ir_is_empty, reduction_code, f);
1350  reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code, f);
1351  translate_function(reduction_code.ir_reduce_one_entry_idx.get(),
1352  ir_reduce_one_entry_idx,
1353  reduction_code,
1354  f);
1356  reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code, f);
1357  reduction_code.llvm_reduce_loop = ir_reduce_loop;
1358  reduction_code.module = std::move(module);
1359  return reduction_code;
1360 }
GroupValueInfo get_group_value_reduction(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const size_t key_width, const QueryMemoryDescriptor &query_mem_desc, const int64_t *that_buff_i64, const size_t that_entry_idx, const size_t that_entry_count, const uint32_t row_size_quad)
void emit_aggregate_one_nullable_value(const std::string &agg_kind, Value *val_ptr, Value *other_ptr, const int64_t init_val, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
std::unique_ptr< CgenState > cgen_state
void clear()
Definition: LruCache.hpp:60
#define CHECK_EQ(x, y)
Definition: Logger.h:205
void reduceOneSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const size_t init_agg_val_idx, const size_t first_slot_idx_for_target, Function *ir_reduce_one_entry) const
bool is_aggregate_query(const QueryDescriptionType hash_type)
void count_distinct_set_union(const int64_t new_set_handle, const int64_t old_set_handle, const CountDistinctDescriptor &new_count_distinct_desc, const CountDistinctDescriptor &old_count_distinct_desc)
__device__ bool dynamic_watchdog()
#define EMPTY_KEY_64
const std::string & label() const
std::unique_ptr< llvm::Module > runtime_module_shallow_copy(CgenState *cgen_state)
RUNTIME_EXPORT uint8_t check_watchdog_rt(const size_t sample_seed)
void reduceOneEntryNoCollisions(const ReductionCode &reduction_code) const
std::shared_ptr< CompilationContext > compilation_context
int64_t getTargetGroupbyIndex(const size_t target_idx) const
void varlen_buffer_sample(int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val)
std::unique_ptr< Function > ir_reduce_loop
Value * emit_read_int_from_buff(Value *ptr, const int8_t compact_sz, Function *function)
void reduceOneEntryBaselineIdx(const ReductionCode &reduction_code) const
SQLTypeInfo sql_type
Definition: TargetInfo.h:42
#define LOG(tag)
Definition: Logger.h:188
void mark_function_always_inline(llvm::Function *func)
void reduceLoop(const ReductionCode &reduction_code) const
bool is_varlen() const
Definition: sqltypes.h:513
string name
Definition: setup.in.py:62
std::string join(T const &container, std::string const &delim)
llvm::Function * llvm_reduce_loop
void reduceOneEntryNoCollisionsIdx(const ReductionCode &reduction_code) const
#define CHECK_GE(x, y)
Definition: Logger.h:210
std::vector< std::string > CodeCacheKey
Definition: CodeCache.h:25
ReductionCode finalizeReductionCode(ReductionCode reduction_code, const llvm::Function *ir_is_empty, const llvm::Function *ir_reduce_one_entry, const llvm::Function *ir_reduce_one_entry_idx, const CodeCacheKey &key) const
size_t get_slot_off_quad(const QueryMemoryDescriptor &query_mem_desc)
std::string cacheKey() const
size_t getEffectiveKeyWidth() const
std::unique_ptr< Function > ir_reduce_one_entry
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:77
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co)
const std::vector< int64_t > target_init_vals_
void reduceOneAggregateSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const int64_t init_val, const int8_t chosen_bytes, Function *ir_reduce_one_entry) const
bool takes_float_argument(const TargetInfo &target_info)
Definition: TargetInfo.h:134
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:115
Value * add(Args &&...args)
bool skip_null_val
Definition: TargetInfo.h:44
const Value * emit_checked_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
std::unique_ptr< Function > setup_reduce_one_entry_idx(ReductionCode *reduction_code)
int32_t(*)(int8_t *this_buff, const int8_t *that_buff, const int32_t start_entry_index, const int32_t end_entry_index, const int32_t that_entry_count, const void *this_qmd, const void *that_qmd, const void *serialized_varlen_buffer) FuncPtr
const QueryMemoryDescriptor query_mem_desc_
std::unique_ptr< Function > ir_is_empty
std::string to_string(char const *&&v)
SQLTypeInfo agg_arg_type
Definition: TargetInfo.h:43
void translate_function(const Function *function, llvm::Function *llvm_function, const ReductionCode &reduction_code, const std::unordered_map< const Function *, llvm::Function * > &f)
void emit_aggregate_one_value(const std::string &agg_kind, Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
RUNTIME_EXPORT void serialized_varlen_buffer_sample(const void *serialized_varlen_buffer_handle, int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val, const int64_t length_to_elems)
Value * emit_load_i32(Value *ptr, Function *function)
Definition: sqldefs.h:73
const SQLTypeInfo get_compact_type(const TargetInfo &target)
__device__ bool check_interrupt()
int8_t get_width_for_slot(const size_t target_slot_idx, const bool float_argument_input, const QueryMemoryDescriptor &query_mem_desc)
llvm::Module * module_
Definition: CgenState.h:318
ResultSetReductionJIT(const QueryMemoryDescriptor &query_mem_desc, const std::vector< TargetInfo > &targets, const std::vector< int64_t > &target_init_vals)
llvm::LLVMContext & context_
Definition: CgenState.h:327
size_t get_byteoff_of_slot(const size_t slot_idx, const QueryMemoryDescriptor &query_mem_desc)
bool is_agg
Definition: TargetInfo.h:40
size_t advance_slot(const size_t j, const TargetInfo &target_info, const bool separate_varlen_storage)
void reduceOneCountDistinctSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
uint8_t check_interrupt_rt(const size_t sample_seed)
size_t getGroupbyColCount() const
size_t targetGroupbyIndicesSize() const
void generate_loop_body(For *for_loop, Function *ir_reduce_loop, Function *ir_reduce_one_entry_idx, Value *this_buff, Value *that_buff, Value *start_index, Value *that_entry_count, Value *this_qmd_handle, Value *that_qmd_handle, Value *serialized_varlen_buffer)
void emit_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
std::unique_ptr< llvm::Module > g_rt_module
Definition: sqldefs.h:75
static std::mutex s_reduction_mutex
bool is_distinct_target(const TargetInfo &target_info)
Definition: TargetInfo.h:130
std::string target_info_key(const TargetInfo &target_info)
std::unique_ptr< Function > ir_reduce_one_entry_idx
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
ReductionCode setup_functions_ir(const QueryDescriptionType hash_type)
DEVICE void allocate()
Definition: quantile.h:579
#define AUTOMATIC_IR_METADATA(CGENSTATE)
std::unique_ptr< Function > setup_is_empty_entry(ReductionCode *reduction_code)
SQLAgg agg_kind
Definition: TargetInfo.h:41
size_t getCountDistinctDescriptorsSize() const
void reduceOneEntryTargetsNoCollisions(Function *ir_reduce_one_entry, Value *this_targets_start_ptr, Value *that_targets_start_ptr) const
QueryDescriptionType getQueryDescriptionType() const
static void addCodeToCache(const CodeCacheKey &, std::shared_ptr< CompilationContext >, llvm::Module *, CodeCache &)
#define AUTOMATIC_IR_METADATA_DONE()
#define UNLIKELY(x)
Definition: likely.h:25
#define RUNTIME_EXPORT
llvm::Type * llvm_type(const Type type, llvm::LLVMContext &ctx)
virtual ReductionCode codegen() const
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
#define CHECK_LT(x, y)
Definition: Logger.h:207
std::string serialize_llvm_object(const T *llvm_obj)
void reduceOneApproxMedianSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
RUNTIME_EXPORT void count_distinct_set_union_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
virtual ReductionCode codegen() const
Definition: sqldefs.h:76
std::unique_ptr< Function > create_function(const std::string name, const std::vector< Function::NamedArg > &arg_types, const Type ret_type, const bool always_inline)
std::string get_type_name() const
Definition: sqltypes.h:417
const Value * iter() const
RUNTIME_EXPORT void get_group_value_reduction_rt(int8_t *groups_buffer, const int8_t *key, const uint32_t key_count, const void *this_qmd_handle, const int8_t *that_buff, const uint32_t that_entry_idx, const uint32_t that_entry_count, const uint32_t row_size_bytes, int64_t **buff_out, uint8_t *empty)
std::unique_ptr< Function > setup_reduce_one_entry(ReductionCode *reduction_code, const QueryDescriptionType hash_type)
void isEmpty(const ReductionCode &reduction_code) const
Value * emit_load_i64(Value *ptr, Function *function)
const ColSlotContext & getColSlotContext() const
#define CHECK(condition)
Definition: Logger.h:197
bool is_geometry() const
Definition: sqltypes.h:500
#define EMPTY_KEY_32
std::unique_ptr< llvm::Module > module
void reduceOneEntryBaseline(const ReductionCode &reduction_code) const
QueryDescriptionType
Definition: Types.h:26
RUNTIME_EXPORT void approx_median_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
char * f
Value * emit_load(Value *ptr, Type ptr_type, Function *function)
bool is_string() const
Definition: sqltypes.h:488
value_t * get(const key_t &key)
Definition: LruCache.hpp:39
llvm::Function * create_llvm_function(const Function *function, CgenState *cgen_state)
bool is_distinct
Definition: TargetInfo.h:45
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:321
void emit_aggregate_one_count(Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, Function *ir_reduce_one_entry)
Definition: sqldefs.h:74
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:712
Definition: sqldefs.h:72
std::unique_ptr< Function > setup_reduce_loop(ReductionCode *reduction_code)
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
std::string reductionKey() const
const Executor * getExecutor() const
int32_t getTargetIdxForKey() const
const std::vector< TargetInfo > targets_