OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ResultSetReductionJIT.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "ResultSetReductionJIT.h"
20 
21 #include "CodeGenerator.h"
22 #include "DynamicWatchdog.h"
23 #include "Execute.h"
24 #include "IRCodegenUtils.h"
27 #include "Shared/StringTransform.h"
28 #include "Shared/likely.h"
29 #include "Shared/quantile.h"
30 
31 #include <llvm/Bitcode/BitcodeReader.h>
32 #include <llvm/IR/Function.h>
33 #include <llvm/IR/IRBuilder.h>
34 #include <llvm/IR/Verifier.h>
35 #include <llvm/Support/SourceMgr.h>
36 #include <llvm/Support/raw_os_ostream.h>
37 
38 namespace {
39 
40 // Error code to be returned when the watchdog timer triggers during the reduction.
41 const int32_t WATCHDOG_ERROR{-1};
42 // Error code to be returned when the interrupt is triggered during the reduction.
43 const int32_t INTERRUPT_ERROR{10};
44 // Use the interpreter, not the JIT, for a number of entries lower than the threshold.
45 const size_t INTERP_THRESHOLD{25};
46 
47 // Load the value stored at 'ptr' interpreted as 'ptr_type'.
48 Value* emit_load(Value* ptr, Type ptr_type, Function* function) {
49  return function->add<Load>(
50  function->add<Cast>(Cast::CastOp::BitCast, ptr, ptr_type, ""),
51  ptr->label() + "_loaded");
52 }
53 
54 // Load the value stored at 'ptr' as a 32-bit signed integer.
55 Value* emit_load_i32(Value* ptr, Function* function) {
56  return emit_load(ptr, Type::Int32Ptr, function);
57 }
58 
59 // Load the value stored at 'ptr' as a 64-bit signed integer.
60 Value* emit_load_i64(Value* ptr, Function* function) {
61  return emit_load(ptr, Type::Int64Ptr, function);
62 }
63 
64 // Read a 32- or 64-bit integer stored at 'ptr' and sign extend to 64-bit.
65 Value* emit_read_int_from_buff(Value* ptr, const int8_t compact_sz, Function* function) {
66  switch (compact_sz) {
67  case 8: {
68  return emit_load_i64(ptr, function);
69  }
70  case 4: {
71  const auto loaded_val = emit_load_i32(ptr, function);
72  return function->add<Cast>(Cast::CastOp::SExt, loaded_val, Type::Int64, "");
73  }
74  default: {
75  LOG(FATAL) << "Invalid byte width: " << compact_sz;
76  return nullptr;
77  }
78  }
79 }
80 
81 // Emit a runtime call to accumulate into the 'val_ptr' byte address the 'other_ptr'
82 // value when the type is specified as not null.
83 void emit_aggregate_one_value(const std::string& agg_kind,
84  Value* val_ptr,
85  Value* other_ptr,
86  const size_t chosen_bytes,
87  const TargetInfo& agg_info,
88  Function* ir_reduce_one_entry) {
89  const auto sql_type = get_compact_type(agg_info);
90  const auto dest_name = agg_kind + "_dest";
91  if (sql_type.is_fp()) {
92  if (chosen_bytes == sizeof(float)) {
93  const auto agg = ir_reduce_one_entry->add<Cast>(
94  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
95  const auto val = emit_load(other_ptr, Type::FloatPtr, ir_reduce_one_entry);
96  ir_reduce_one_entry->add<Call>(
97  "agg_" + agg_kind + "_float", std::vector<const Value*>{agg, val}, "");
98  } else {
99  CHECK_EQ(chosen_bytes, sizeof(double));
100  const auto agg = ir_reduce_one_entry->add<Cast>(
101  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
102  const auto val = emit_load(other_ptr, Type::DoublePtr, ir_reduce_one_entry);
103  ir_reduce_one_entry->add<Call>(
104  "agg_" + agg_kind + "_double", std::vector<const Value*>{agg, val}, "");
105  }
106  } else {
107  if (chosen_bytes == sizeof(int32_t)) {
108  const auto agg = ir_reduce_one_entry->add<Cast>(
109  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
110  const auto val = emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
111  ir_reduce_one_entry->add<Call>(
112  "agg_" + agg_kind + "_int32", std::vector<const Value*>{agg, val}, "");
113  } else {
114  CHECK_EQ(chosen_bytes, sizeof(int64_t));
115  const auto agg = ir_reduce_one_entry->add<Cast>(
116  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
117  const auto val = emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
118  ir_reduce_one_entry->add<Call>(
119  "agg_" + agg_kind, std::vector<const Value*>{agg, val}, "");
120  }
121  }
122 }
123 
124 // Same as above, but support nullable types as well.
126  Value* val_ptr,
127  Value* other_ptr,
128  const int64_t init_val,
129  const size_t chosen_bytes,
130  const TargetInfo& agg_info,
131  Function* ir_reduce_one_entry) {
132  const std::string agg_kind = to_lower(toString(get_non_conditional_agg_type(sql_agg)));
133  const auto dest_name = agg_kind + "_dest";
134  if (agg_info.skip_null_val) {
135  const auto sql_type = get_compact_type(agg_info);
136  if (sql_type.is_fp()) {
137  if (chosen_bytes == sizeof(float)) {
138  const auto agg = ir_reduce_one_entry->add<Cast>(
139  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
140  const auto val = emit_load(other_ptr, Type::FloatPtr, ir_reduce_one_entry);
141  const auto init_val_lv = ir_reduce_one_entry->addConstant<ConstantFP>(
142  *reinterpret_cast<const float*>(may_alias_ptr(&init_val)), Type::Float);
143  std::vector<const Value*> args{agg, val, init_val_lv};
144  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_float_skip_val", args, "");
145  } else {
146  CHECK_EQ(chosen_bytes, sizeof(double));
147  const auto agg = ir_reduce_one_entry->add<Cast>(
148  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
149  const auto val = emit_load(other_ptr, Type::DoublePtr, ir_reduce_one_entry);
150  const auto init_val_lv = ir_reduce_one_entry->addConstant<ConstantFP>(
151  *reinterpret_cast<const double*>(may_alias_ptr(&init_val)), Type::Double);
152  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_double_skip_val",
153  std::vector<const Value*>{agg, val, init_val_lv},
154  "");
155  }
156  } else {
157  if (chosen_bytes == sizeof(int32_t)) {
158  const auto agg = ir_reduce_one_entry->add<Cast>(
159  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
160  const auto val = emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
161  const auto init_val_lv =
162  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int32);
163  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_int32_skip_val",
164  std::vector<const Value*>{agg, val, init_val_lv},
165  "");
166  } else {
167  CHECK_EQ(chosen_bytes, sizeof(int64_t));
168  const auto agg = ir_reduce_one_entry->add<Cast>(
169  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
170  const auto val = emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
171  const auto init_val_lv =
172  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64);
173  ir_reduce_one_entry->add<Call>("agg_" + agg_kind + "_skip_val",
174  std::vector<const Value*>{agg, val, init_val_lv},
175  "");
176  }
177  }
178  } else {
180  agg_kind, val_ptr, other_ptr, chosen_bytes, agg_info, ir_reduce_one_entry);
181  }
182 }
183 
184 // Emit code to accumulate the 'other_ptr' count into the 'val_ptr' destination.
186  Value* other_ptr,
187  const size_t chosen_bytes,
188  Function* ir_reduce_one_entry) {
189  const auto dest_name = "count_dest";
190  if (chosen_bytes == sizeof(int32_t)) {
191  const auto agg = ir_reduce_one_entry->add<Cast>(
192  Cast::CastOp::BitCast, val_ptr, Type::Int32Ptr, dest_name);
193  const auto val = emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
194  ir_reduce_one_entry->add<Call>(
195  "agg_sum_int32", std::vector<const Value*>{agg, val}, "");
196  } else {
197  CHECK_EQ(chosen_bytes, sizeof(int64_t));
198  const auto agg = ir_reduce_one_entry->add<Cast>(
199  Cast::CastOp::BitCast, val_ptr, Type::Int64Ptr, dest_name);
200  const auto val = emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
201  ir_reduce_one_entry->add<Call>("agg_sum", std::vector<const Value*>{agg, val}, "");
202  }
203 }
204 
205 // Emit code to load the value stored at the 'other_pi8' as an integer of the given width
206 // 'chosen_bytes' and write it to the 'slot_pi8' destination only if necessary (the
207 // existing value at destination is the initialization value).
209  Value* other_pi8,
210  const int64_t init_val,
211  const size_t chosen_bytes,
212  Function* ir_reduce_one_entry) {
213  const auto func_name = "write_projection_int" + std::to_string(chosen_bytes * 8);
214  if (chosen_bytes == sizeof(int32_t)) {
215  const auto proj_val = emit_load_i32(other_pi8, ir_reduce_one_entry);
216  ir_reduce_one_entry->add<Call>(
217  func_name,
218  std::vector<const Value*>{
219  slot_pi8,
220  proj_val,
221  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64)},
222  "");
223  } else {
224  CHECK_EQ(chosen_bytes, sizeof(int64_t));
225  const auto proj_val = emit_load_i64(other_pi8, ir_reduce_one_entry);
226  ir_reduce_one_entry->add<Call>(
227  func_name,
228  std::vector<const Value*>{
229  slot_pi8,
230  proj_val,
231  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64)},
232  "");
233  }
234 }
235 
236 // Emit code to load the value stored at the 'other_pi8' as an integer of the given width
237 // 'chosen_bytes' and write it to the 'slot_pi8' destination only if necessary (the
238 // existing value at destination is the initialization value).
240  Value* other_pi8,
241  const int64_t init_val,
242  const size_t chosen_bytes,
243  Function* ir_reduce_one_entry) {
244  if (chosen_bytes == sizeof(int32_t)) {
245  const auto func_name = "checked_single_agg_id_int32";
246  const auto proj_val = emit_load_i32(other_pi8, ir_reduce_one_entry);
247  const auto slot_pi32 = ir_reduce_one_entry->add<Cast>(
248  Cast::CastOp::BitCast, slot_pi8, Type::Int32Ptr, "");
249  return ir_reduce_one_entry->add<Call>(
250  func_name,
251  Type::Int32,
252  std::vector<const Value*>{
253  slot_pi32,
254  proj_val,
255  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int32)},
256  "");
257  } else {
258  const auto func_name = "checked_single_agg_id";
259  CHECK_EQ(chosen_bytes, sizeof(int64_t));
260  const auto proj_val = emit_load_i64(other_pi8, ir_reduce_one_entry);
261  const auto slot_pi64 = ir_reduce_one_entry->add<Cast>(
262  Cast::CastOp::BitCast, slot_pi8, Type::Int64Ptr, "");
263 
264  return ir_reduce_one_entry->add<Call>(
265  func_name,
266  Type::Int32,
267  std::vector<const Value*>{
268  slot_pi64,
269  proj_val,
270  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64)},
271  "");
272  }
273 }
274 
275 std::unique_ptr<Function> create_function(
276  const std::string name,
277  const std::vector<Function::NamedArg>& arg_types,
278  const Type ret_type,
279  const bool always_inline) {
280  return std::make_unique<Function>(name, arg_types, ret_type, always_inline);
281 }
282 
283 // Create the declaration for the 'is_empty_entry' function. Use private linkage since
284 // it's a helper only called from the generated code and mark it as always inline.
285 std::unique_ptr<Function> setup_is_empty_entry(ReductionCode* reduction_code) {
286  return create_function(
287  "is_empty_entry", {{"row_ptr", Type::Int8Ptr}}, Type::Int1, /*always_inline=*/true);
288 }
289 
290 // Create the declaration for the 'reduce_one_entry' helper.
291 std::unique_ptr<Function> setup_reduce_one_entry(ReductionCode* reduction_code,
292  const QueryDescriptionType hash_type) {
293  std::string this_ptr_name;
294  std::string that_ptr_name;
295  switch (hash_type) {
297  this_ptr_name = "this_targets_ptr";
298  that_ptr_name = "that_targets_ptr";
299  break;
300  }
303  this_ptr_name = "this_row_ptr";
304  that_ptr_name = "that_row_ptr";
305  break;
306  }
307  default: {
308  LOG(FATAL) << "Unexpected query description type";
309  }
310  }
311  return create_function("reduce_one_entry",
312  {{this_ptr_name, Type::Int8Ptr},
313  {that_ptr_name, Type::Int8Ptr},
314  {"this_qmd", Type::VoidPtr},
315  {"that_qmd", Type::VoidPtr},
316  {"serialized_varlen_buffer_arg", Type::VoidPtr}},
317  Type::Int32,
318  /*always_inline=*/true);
319 }
320 
321 // Create the declaration for the 'reduce_one_entry_idx' helper.
322 std::unique_ptr<Function> setup_reduce_one_entry_idx(ReductionCode* reduction_code) {
323  return create_function("reduce_one_entry_idx",
324  {{"this_buff", Type::Int8Ptr},
325  {"that_buff", Type::Int8Ptr},
326  {"that_entry_idx", Type::Int32},
327  {"that_entry_count", Type::Int32},
328  {"this_qmd_handle", Type::VoidPtr},
329  {"that_qmd_handle", Type::VoidPtr},
330  {"serialized_varlen_buffer", Type::VoidPtr}},
331  Type::Int32,
332  /*always_inline=*/true);
333 }
334 
335 // Create the declaration for the 'reduce_loop' entry point. Use external linkage, this is
336 // the public API of the generated code directly used from result set reduction.
337 std::unique_ptr<Function> setup_reduce_loop(ReductionCode* reduction_code) {
338  return create_function("reduce_loop",
339  {{"this_buff", Type::Int8Ptr},
340  {"that_buff", Type::Int8Ptr},
341  {"start_index", Type::Int32},
342  {"end_index", Type::Int32},
343  {"that_entry_count", Type::Int32},
344  {"this_qmd_handle", Type::VoidPtr},
345  {"that_qmd_handle", Type::VoidPtr},
346  {"serialized_varlen_buffer", Type::VoidPtr}},
347  Type::Int32,
348  /*always_inline=*/false);
349 }
350 
351 llvm::Function* create_llvm_function(const Function* function, CgenState* cgen_state) {
352  AUTOMATIC_IR_METADATA(cgen_state);
353  auto& ctx = cgen_state->context_;
354  std::vector<llvm::Type*> parameter_types;
355  const auto& arg_types = function->arg_types();
356  for (const auto& named_arg : arg_types) {
357  CHECK(named_arg.type != Type::Void);
358  parameter_types.push_back(llvm_type(named_arg.type, ctx));
359  }
360  const auto func_type = llvm::FunctionType::get(
361  llvm_type(function->ret_type(), ctx), parameter_types, false);
362  const auto linkage = function->always_inline() ? llvm::Function::PrivateLinkage
363  : llvm::Function::ExternalLinkage;
364  auto func =
365  llvm::Function::Create(func_type, linkage, function->name(), cgen_state->module_);
366  const auto arg_it = func->arg_begin();
367  for (size_t i = 0; i < arg_types.size(); ++i) {
368  const auto arg = &*(arg_it + i);
369  arg->setName(arg_types[i].name);
370  }
371  if (function->always_inline()) {
373  }
374  return func;
375 }
376 
377 // Setup the reduction function and helpers declarations, create a module and a code
378 // generation state object.
380  ReductionCode reduction_code{};
381  reduction_code.ir_is_empty = setup_is_empty_entry(&reduction_code);
382  reduction_code.ir_reduce_one_entry = setup_reduce_one_entry(&reduction_code, hash_type);
383  reduction_code.ir_reduce_one_entry_idx = setup_reduce_one_entry_idx(&reduction_code);
384  reduction_code.ir_reduce_loop = setup_reduce_loop(&reduction_code);
385  return reduction_code;
386 }
387 
389  return hash_type == QueryDescriptionType::GroupByBaselineHash ||
392 }
393 
394 // Variable length sample fast path (no serialized variable length buffer).
395 void varlen_buffer_sample(int8_t* this_ptr1,
396  int8_t* this_ptr2,
397  const int8_t* that_ptr1,
398  const int8_t* that_ptr2,
399  const int64_t init_val) {
400  const auto rhs_proj_col = *reinterpret_cast<const int64_t*>(that_ptr1);
401  if (rhs_proj_col != init_val) {
402  *reinterpret_cast<int64_t*>(this_ptr1) = rhs_proj_col;
403  }
404  CHECK(this_ptr2 && that_ptr2);
405  *reinterpret_cast<int64_t*>(this_ptr2) = *reinterpret_cast<const int64_t*>(that_ptr2);
406 }
407 
408 } // namespace
409 
411  const void* serialized_varlen_buffer_handle,
412  int8_t* this_ptr1,
413  int8_t* this_ptr2,
414  const int8_t* that_ptr1,
415  const int8_t* that_ptr2,
416  const int64_t init_val,
417  const int64_t length_to_elems) {
418  if (!serialized_varlen_buffer_handle) {
419  varlen_buffer_sample(this_ptr1, this_ptr2, that_ptr1, that_ptr2, init_val);
420  return;
421  }
422  const auto& serialized_varlen_buffer =
423  *reinterpret_cast<const std::vector<std::string>*>(serialized_varlen_buffer_handle);
424  if (!serialized_varlen_buffer.empty()) {
425  const auto rhs_proj_col = *reinterpret_cast<const int64_t*>(that_ptr1);
426  CHECK_LT(static_cast<size_t>(rhs_proj_col), serialized_varlen_buffer.size());
427  const auto& varlen_bytes_str = serialized_varlen_buffer[rhs_proj_col];
428  const auto str_ptr = reinterpret_cast<const int8_t*>(varlen_bytes_str.c_str());
429  *reinterpret_cast<int64_t*>(this_ptr1) = reinterpret_cast<const int64_t>(str_ptr);
430  *reinterpret_cast<int64_t*>(this_ptr2) =
431  static_cast<int64_t>(varlen_bytes_str.size() / length_to_elems);
432  } else {
433  varlen_buffer_sample(this_ptr1, this_ptr2, that_ptr1, that_ptr2, init_val);
434  }
435 }
436 
437 // Wrappers to be called from the generated code, sharing implementation with the rest of
438 // the system.
439 
441  const int64_t new_set_handle,
442  const int64_t old_set_handle,
443  const void* that_qmd_handle,
444  const void* this_qmd_handle,
445  const int64_t target_logical_idx) {
446  const auto that_qmd = reinterpret_cast<const QueryMemoryDescriptor*>(that_qmd_handle);
447  const auto this_qmd = reinterpret_cast<const QueryMemoryDescriptor*>(this_qmd_handle);
448  const auto& new_count_distinct_desc =
449  that_qmd->getCountDistinctDescriptor(target_logical_idx);
450  const auto& old_count_distinct_desc =
451  this_qmd->getCountDistinctDescriptor(target_logical_idx);
452  CHECK(old_count_distinct_desc.impl_type_ != CountDistinctImplType::Invalid);
453  CHECK(old_count_distinct_desc.impl_type_ == new_count_distinct_desc.impl_type_);
455  new_set_handle, old_set_handle, new_count_distinct_desc, old_count_distinct_desc);
456 }
457 
458 extern "C" RUNTIME_EXPORT void approx_quantile_jit_rt(const int64_t new_set_handle,
459  const int64_t old_set_handle,
460  const void* that_qmd_handle,
461  const void* this_qmd_handle,
462  const int64_t target_logical_idx) {
463  auto* incoming = reinterpret_cast<quantile::TDigest*>(new_set_handle);
464  if (incoming->centroids().capacity()) {
465  auto* accumulator = reinterpret_cast<quantile::TDigest*>(old_set_handle);
466  accumulator->allocate();
467  accumulator->mergeTDigest(*incoming);
468  }
469 }
470 
471 extern "C" RUNTIME_EXPORT void mode_jit_rt(const int64_t new_set_handle,
472  const int64_t old_set_handle,
473  const void* that_qmd_handle,
474  const void* this_qmd_handle,
475  const int64_t target_logical_idx) {
476  auto* accumulator = reinterpret_cast<AggMode*>(old_set_handle);
477  auto* incoming = reinterpret_cast<AggMode*>(new_set_handle);
478  accumulator->reduce(std::move(*incoming));
479 }
480 
482  int8_t* groups_buffer,
483  const int8_t* key,
484  const uint32_t key_count,
485  const void* this_qmd_handle,
486  const int8_t* that_buff,
487  const uint32_t that_entry_idx,
488  const uint32_t that_entry_count,
489  const uint32_t row_size_bytes,
490  int64_t** buff_out,
491  uint8_t* empty) {
492  const auto& this_qmd = *reinterpret_cast<const QueryMemoryDescriptor*>(this_qmd_handle);
493  const auto gvi =
494  result_set::get_group_value_reduction(reinterpret_cast<int64_t*>(groups_buffer),
495  this_qmd.getEntryCount(),
496  reinterpret_cast<const int64_t*>(key),
497  key_count,
498  this_qmd.getEffectiveKeyWidth(),
499  this_qmd,
500  reinterpret_cast<const int64_t*>(that_buff),
501  that_entry_idx,
502  that_entry_count,
503  row_size_bytes >> 3);
504  *buff_out = gvi.first;
505  *empty = gvi.second;
506 }
507 
508 extern "C" RUNTIME_EXPORT uint8_t check_watchdog_rt(const size_t sample_seed) {
509  if (UNLIKELY(g_enable_dynamic_watchdog && (sample_seed & 0x3F) == 0 &&
510  dynamic_watchdog())) {
511  return true;
512  }
513  return false;
514 }
515 
516 extern "C" uint8_t check_interrupt_rt(const size_t sample_seed) {
517  // this func is called iff we enable runtime query interrupt
518  if (UNLIKELY((sample_seed & 0xFFFF) == 0 && check_interrupt())) {
519  return true;
520  }
521  return false;
522 }
523 
525  const std::vector<TargetInfo>& targets,
526  const std::vector<int64_t>& target_init_vals,
527  const size_t executor_id)
528  : executor_id_(executor_id)
529  , query_mem_desc_(query_mem_desc)
530  , targets_(targets)
531  , target_init_vals_(target_init_vals) {}
532 
533 // The code generated for a reduction between two result set buffers is structured in
534 // several functions and their IR is stored in the 'ReductionCode' structure. At a high
535 // level, the pseudocode is:
536 //
537 // func is_empty_func(row_ptr):
538 // ...
539 //
540 // func reduce_func_baseline(this_ptr, that_ptr):
541 // if is_empty_func(that_ptr):
542 // return
543 // for each target in the row:
544 // reduce target from that_ptr into this_ptr
545 //
546 // func reduce_func_perfect_hash(this_ptr, that_ptr):
547 // if is_empty_func(that_ptr):
548 // return
549 // for each target in the row:
550 // reduce target from that_ptr into this_ptr
551 //
552 // func reduce_func_idx(this_buff, that_buff, that_entry_index):
553 // that_ptr = that_result_set[that_entry_index]
554 // # Retrieval of 'this_ptr' is different between perfect hash and baseline.
555 // this_ptr = this_result_set[that_entry_index]
556 // or
557 // get_row(key(that_row_ptr), this_result_setBuffer)
558 // reduce_func_[baseline|perfect_hash](this_ptr, that_ptr)
559 //
560 // func reduce_loop(this_buff, that_buff, start_entry_index, end_entry_index):
561 // for that_entry_index in [start_entry_index, end_entry_index):
562 // reduce_func_idx(this_buff, that_buff, that_entry_index)
563 
565  const auto hash_type = query_mem_desc_.getQueryDescriptionType();
567  return {};
568  }
569  auto reduction_code = setup_functions_ir(hash_type);
570  isEmpty(reduction_code);
574  reduceOneEntryNoCollisions(reduction_code);
575  reduceOneEntryNoCollisionsIdx(reduction_code);
576  break;
577  }
579  reduceOneEntryBaseline(reduction_code);
580  reduceOneEntryBaselineIdx(reduction_code);
581  break;
582  }
583  default: {
584  LOG(FATAL) << "Unexpected query description type";
585  }
586  }
587  reduceLoop(reduction_code);
588  // For small result sets, avoid native code generation and use the interpreter instead.
591  return reduction_code;
592  }
593  auto executor = Executor::getExecutor(executor_id_);
594  CodeCacheKey key{cacheKey()};
595  std::lock_guard<std::mutex> compilation_lock(executor->compilation_mutex_);
596  const auto compilation_context =
597  QueryEngine::getInstance()->s_code_accessor->get_or_wait(key);
598  if (compilation_context) {
599  reduction_code.func_ptr =
600  reinterpret_cast<ReductionCode::FuncPtr>(compilation_context->get()->func());
601  return reduction_code;
602  }
603  auto cgen_state_ = std::unique_ptr<CgenState>(new CgenState({}, false, executor.get()));
604  auto cgen_state = reduction_code.cgen_state = cgen_state_.get();
605  cgen_state->set_module_shallow_copy(executor->get_rt_module());
606  reduction_code.module = cgen_state->module_;
607 
608  AUTOMATIC_IR_METADATA(cgen_state);
609  auto ir_is_empty = create_llvm_function(reduction_code.ir_is_empty.get(), cgen_state);
610  auto ir_reduce_one_entry =
611  create_llvm_function(reduction_code.ir_reduce_one_entry.get(), cgen_state);
612  auto ir_reduce_one_entry_idx =
613  create_llvm_function(reduction_code.ir_reduce_one_entry_idx.get(), cgen_state);
614  auto ir_reduce_loop =
615  create_llvm_function(reduction_code.ir_reduce_loop.get(), cgen_state);
616  std::unordered_map<const Function*, llvm::Function*> f;
617  f.emplace(reduction_code.ir_is_empty.get(), ir_is_empty);
618  f.emplace(reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry);
619  f.emplace(reduction_code.ir_reduce_one_entry_idx.get(), ir_reduce_one_entry_idx);
620  f.emplace(reduction_code.ir_reduce_loop.get(), ir_reduce_loop);
621  translate_function(reduction_code.ir_is_empty.get(), ir_is_empty, reduction_code, f);
623  reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code, f);
624  translate_function(reduction_code.ir_reduce_one_entry_idx.get(),
625  ir_reduce_one_entry_idx,
626  reduction_code,
627  f);
629  reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code, f);
630  reduction_code.llvm_reduce_loop = ir_reduce_loop;
632  reduction_code.cgen_state = nullptr;
634  reduction_code, ir_is_empty, ir_reduce_one_entry, ir_reduce_one_entry_idx, key);
635  return reduction_code;
636 }
637 
638 void ResultSetReductionJIT::isEmpty(const ReductionCode& reduction_code) const {
639  auto ir_is_empty = reduction_code.ir_is_empty.get();
642  Value* key{nullptr};
643  Value* empty_key_val{nullptr};
644  const auto keys_ptr = ir_is_empty->arg(0);
649  CHECK_LT(static_cast<size_t>(query_mem_desc_.getTargetIdxForKey()),
650  target_init_vals_.size());
651  const int64_t target_slot_off = result_set::get_byteoff_of_slot(
653  const auto slot_ptr = ir_is_empty->add<GetElementPtr>(
654  keys_ptr,
655  ir_is_empty->addConstant<ConstantInt>(target_slot_off, Type::Int32),
656  "is_empty_slot_ptr");
657  const auto compact_sz =
659  key = emit_read_int_from_buff(slot_ptr, compact_sz, ir_is_empty);
660  empty_key_val = ir_is_empty->addConstant<ConstantInt>(
662  } else {
664  case 4: {
667  key = emit_load_i32(keys_ptr, ir_is_empty);
668  empty_key_val = ir_is_empty->addConstant<ConstantInt>(EMPTY_KEY_32, Type::Int32);
669  break;
670  }
671  case 8: {
672  key = emit_load_i64(keys_ptr, ir_is_empty);
673  empty_key_val = ir_is_empty->addConstant<ConstantInt>(EMPTY_KEY_64, Type::Int64);
674  break;
675  }
676  default:
677  LOG(FATAL) << "Invalid key width";
678  }
679  }
680  const auto ret =
681  ir_is_empty->add<ICmp>(ICmp::Predicate::EQ, key, empty_key_val, "is_key_empty");
682  ir_is_empty->add<Ret>(ret);
683 }
684 
686  const ReductionCode& reduction_code) const {
687  auto ir_reduce_one_entry = reduction_code.ir_reduce_one_entry.get();
688  const auto this_row_ptr = ir_reduce_one_entry->arg(0);
689  const auto that_row_ptr = ir_reduce_one_entry->arg(1);
690  const auto that_is_empty =
691  ir_reduce_one_entry->add<Call>(reduction_code.ir_is_empty.get(),
692  std::vector<const Value*>{that_row_ptr},
693  "that_is_empty");
694  ir_reduce_one_entry->add<ReturnEarly>(
695  that_is_empty, ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32), "");
696 
697  const auto key_bytes = get_key_bytes_rowwise(query_mem_desc_);
698  if (key_bytes) { // copy the key from right hand side
699  ir_reduce_one_entry->add<MemCpy>(
700  this_row_ptr,
701  that_row_ptr,
702  ir_reduce_one_entry->addConstant<ConstantInt>(key_bytes, Type::Int32));
703  }
704 
705  const auto key_bytes_with_padding = align_to_int64(key_bytes);
706  const auto key_bytes_lv =
707  ir_reduce_one_entry->addConstant<ConstantInt>(key_bytes_with_padding, Type::Int32);
708  const auto this_targets_start_ptr = ir_reduce_one_entry->add<GetElementPtr>(
709  this_row_ptr, key_bytes_lv, "this_targets_start");
710  const auto that_targets_start_ptr = ir_reduce_one_entry->add<GetElementPtr>(
711  that_row_ptr, key_bytes_lv, "that_targets_start");
712 
714  ir_reduce_one_entry, this_targets_start_ptr, that_targets_start_ptr);
715 }
716 
718  Function* ir_reduce_one_entry,
719  Value* this_targets_start_ptr,
720  Value* that_targets_start_ptr) const {
721  const auto& col_slot_context = query_mem_desc_.getColSlotContext();
722  Value* this_targets_ptr = this_targets_start_ptr;
723  Value* that_targets_ptr = that_targets_start_ptr;
724  size_t init_agg_val_idx = 0;
725  for (size_t target_logical_idx = 0; target_logical_idx < targets_.size();
726  ++target_logical_idx) {
727  const auto& target_info = targets_[target_logical_idx];
728  const auto& slots_for_col = col_slot_context.getSlotsForCol(target_logical_idx);
729  Value* this_ptr2{nullptr};
730  Value* that_ptr2{nullptr};
731 
732  bool two_slot_target{false};
733  if (target_info.is_agg &&
734  (target_info.agg_kind == kAVG ||
735  (target_info.agg_kind == kSAMPLE && target_info.sql_type.is_varlen()))) {
736  // Note that this assumes if one of the slot pairs in a given target is an array,
737  // all slot pairs are arrays. Currently this is true for all geo targets, but we
738  // should better codify and store this information in the future
739  two_slot_target = true;
740  }
741 
742  for (size_t target_slot_idx = slots_for_col.front();
743  target_slot_idx < slots_for_col.back() + 1;
744  target_slot_idx += 2) {
745  const auto slot_off_val = query_mem_desc_.getPaddedSlotWidthBytes(target_slot_idx);
746  const auto slot_off =
747  ir_reduce_one_entry->addConstant<ConstantInt>(slot_off_val, Type::Int32);
748  if (UNLIKELY(two_slot_target)) {
749  const auto desc = "target_" + std::to_string(target_logical_idx) + "_second_slot";
750  this_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
751  this_targets_ptr, slot_off, "this_" + desc);
752  that_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
753  that_targets_ptr, slot_off, "that_" + desc);
754  }
755  reduceOneSlot(this_targets_ptr,
756  this_ptr2,
757  that_targets_ptr,
758  that_ptr2,
759  target_info,
760  target_logical_idx,
761  target_slot_idx,
762  init_agg_val_idx,
763  slots_for_col.front(),
764  ir_reduce_one_entry);
765  auto increment_agg_val_idx_maybe =
766  [&init_agg_val_idx, &target_logical_idx, this](const int slot_count) {
768  query_mem_desc_.getTargetGroupbyIndex(target_logical_idx) < 0) {
769  init_agg_val_idx += slot_count;
770  }
771  };
772  if (target_logical_idx + 1 == targets_.size() &&
773  target_slot_idx + 1 >= slots_for_col.back()) {
774  break;
775  }
776  const auto next_desc =
777  "target_" + std::to_string(target_logical_idx + 1) + "_first_slot";
778  if (UNLIKELY(two_slot_target)) {
779  increment_agg_val_idx_maybe(2);
780  const auto two_slot_off = ir_reduce_one_entry->addConstant<ConstantInt>(
781  slot_off_val + query_mem_desc_.getPaddedSlotWidthBytes(target_slot_idx + 1),
782  Type::Int32);
783  this_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
784  this_targets_ptr, two_slot_off, "this_" + next_desc);
785  that_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
786  that_targets_ptr, two_slot_off, "that_" + next_desc);
787  } else {
788  increment_agg_val_idx_maybe(1);
789  this_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
790  this_targets_ptr, slot_off, "this_" + next_desc);
791  that_targets_ptr = ir_reduce_one_entry->add<GetElementPtr>(
792  that_targets_ptr, slot_off, "that_" + next_desc);
793  }
794  }
795  }
796  ir_reduce_one_entry->add<Ret>(
797  ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32));
798 }
799 
801  const ReductionCode& reduction_code) const {
802  auto ir_reduce_one_entry = reduction_code.ir_reduce_one_entry.get();
803  const auto this_targets_ptr_arg = ir_reduce_one_entry->arg(0);
804  const auto that_targets_ptr_arg = ir_reduce_one_entry->arg(1);
805  Value* this_ptr1 = this_targets_ptr_arg;
806  Value* that_ptr1 = that_targets_ptr_arg;
807  size_t j = 0;
808  size_t init_agg_val_idx = 0;
809  for (size_t target_logical_idx = 0; target_logical_idx < targets_.size();
810  ++target_logical_idx) {
811  const auto& target_info = targets_[target_logical_idx];
812  Value* this_ptr2{nullptr};
813  Value* that_ptr2{nullptr};
814  if (target_info.is_agg &&
815  (target_info.agg_kind == kAVG ||
816  (target_info.agg_kind == kSAMPLE && target_info.sql_type.is_varlen()))) {
817  const auto desc = "target_" + std::to_string(target_logical_idx) + "_second_slot";
818  const auto second_slot_rel_off =
819  ir_reduce_one_entry->addConstant<ConstantInt>(sizeof(int64_t), Type::Int32);
820  this_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
821  this_ptr1, second_slot_rel_off, "this_" + desc);
822  that_ptr2 = ir_reduce_one_entry->add<GetElementPtr>(
823  that_ptr1, second_slot_rel_off, "that_" + desc);
824  }
825  reduceOneSlot(this_ptr1,
826  this_ptr2,
827  that_ptr1,
828  that_ptr2,
829  target_info,
830  target_logical_idx,
831  j,
832  init_agg_val_idx,
833  j,
834  ir_reduce_one_entry);
835  if (target_logical_idx + 1 == targets_.size()) {
836  break;
837  }
839  init_agg_val_idx = advance_slot(init_agg_val_idx, target_info, false);
840  } else {
841  if (query_mem_desc_.getTargetGroupbyIndex(target_logical_idx) < 0) {
842  init_agg_val_idx = advance_slot(init_agg_val_idx, target_info, false);
843  }
844  }
845  j = advance_slot(j, target_info, false);
846  const auto next_desc =
847  "target_" + std::to_string(target_logical_idx + 1) + "_first_slot";
848  auto next_slot_rel_off = ir_reduce_one_entry->addConstant<ConstantInt>(
849  init_agg_val_idx * sizeof(int64_t), Type::Int32);
850  this_ptr1 = ir_reduce_one_entry->add<GetElementPtr>(
851  this_targets_ptr_arg, next_slot_rel_off, next_desc);
852  that_ptr1 = ir_reduce_one_entry->add<GetElementPtr>(
853  that_targets_ptr_arg, next_slot_rel_off, next_desc);
854  }
855  ir_reduce_one_entry->add<Ret>(
856  ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32));
857 }
858 
860  const ReductionCode& reduction_code) const {
861  auto ir_reduce_one_entry_idx = reduction_code.ir_reduce_one_entry_idx.get();
866  const auto this_buff = ir_reduce_one_entry_idx->arg(0);
867  const auto that_buff = ir_reduce_one_entry_idx->arg(1);
868  const auto entry_idx = ir_reduce_one_entry_idx->arg(2);
869  const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
870  const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
871  const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
872  const auto row_bytes = ir_reduce_one_entry_idx->addConstant<ConstantInt>(
874  const auto entry_idx_64 = ir_reduce_one_entry_idx->add<Cast>(
875  Cast::CastOp::SExt, entry_idx, Type::Int64, "entry_idx_64");
876  const auto row_off_in_bytes = ir_reduce_one_entry_idx->add<BinaryOperator>(
877  BinaryOperator::BinaryOp::Mul, entry_idx_64, row_bytes, "row_off_in_bytes");
878  const auto this_row_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
879  this_buff, row_off_in_bytes, "this_row_ptr");
880  const auto that_row_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
881  that_buff, row_off_in_bytes, "that_row_ptr");
882  const auto reduce_rc = ir_reduce_one_entry_idx->add<Call>(
883  reduction_code.ir_reduce_one_entry.get(),
884  std::vector<const Value*>{this_row_ptr,
885  that_row_ptr,
886  this_qmd_handle,
887  that_qmd_handle,
888  serialized_varlen_buffer_arg},
889  "");
890  ir_reduce_one_entry_idx->add<Ret>(reduce_rc);
891 }
892 
894  const ReductionCode& reduction_code) const {
895  auto ir_reduce_one_entry_idx = reduction_code.ir_reduce_one_entry_idx.get();
900  const auto this_buff = ir_reduce_one_entry_idx->arg(0);
901  const auto that_buff = ir_reduce_one_entry_idx->arg(1);
902  const auto that_entry_idx = ir_reduce_one_entry_idx->arg(2);
903  const auto that_entry_count = ir_reduce_one_entry_idx->arg(3);
904  const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
905  const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
906  const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
907  const auto row_bytes = ir_reduce_one_entry_idx->addConstant<ConstantInt>(
909  const auto that_entry_idx_64 = ir_reduce_one_entry_idx->add<Cast>(
910  Cast::CastOp::SExt, that_entry_idx, Type::Int64, "that_entry_idx_64");
911  const auto that_row_off_in_bytes =
912  ir_reduce_one_entry_idx->add<BinaryOperator>(BinaryOperator::BinaryOp::Mul,
913  that_entry_idx_64,
914  row_bytes,
915  "that_row_off_in_bytes");
916  const auto that_row_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
917  that_buff, that_row_off_in_bytes, "that_row_ptr");
918  const auto that_is_empty =
919  ir_reduce_one_entry_idx->add<Call>(reduction_code.ir_is_empty.get(),
920  std::vector<const Value*>{that_row_ptr},
921  "that_is_empty");
922  ir_reduce_one_entry_idx->add<ReturnEarly>(
923  that_is_empty,
924  ir_reduce_one_entry_idx->addConstant<ConstantInt>(0, Type::Int32),
925  "");
926  const auto key_count = query_mem_desc_.getGroupbyColCount();
927  const auto one_element =
928  ir_reduce_one_entry_idx->addConstant<ConstantInt>(1, Type::Int32);
929  const auto this_targets_ptr_i64_ptr = ir_reduce_one_entry_idx->add<Alloca>(
930  Type::Int64Ptr, one_element, "this_targets_ptr_out");
931  const auto this_is_empty_ptr =
932  ir_reduce_one_entry_idx->add<Alloca>(Type::Int8, one_element, "this_is_empty_out");
933  ir_reduce_one_entry_idx->add<ExternalCall>(
934  "get_group_value_reduction_rt",
935  Type::Void,
936  std::vector<const Value*>{
937  this_buff,
938  that_row_ptr,
939  ir_reduce_one_entry_idx->addConstant<ConstantInt>(key_count, Type::Int32),
940  this_qmd_handle,
941  that_buff,
942  that_entry_idx,
943  that_entry_count,
944  row_bytes,
945  this_targets_ptr_i64_ptr,
946  this_is_empty_ptr},
947  "");
948  const auto this_targets_ptr_i64 = ir_reduce_one_entry_idx->add<Load>(
949  this_targets_ptr_i64_ptr, "this_targets_ptr_i64");
950  auto this_is_empty =
951  ir_reduce_one_entry_idx->add<Load>(this_is_empty_ptr, "this_is_empty");
952  this_is_empty = ir_reduce_one_entry_idx->add<Cast>(
953  Cast::CastOp::Trunc, this_is_empty, Type::Int1, "this_is_empty_bool");
954  ir_reduce_one_entry_idx->add<ReturnEarly>(
955  this_is_empty,
956  ir_reduce_one_entry_idx->addConstant<ConstantInt>(0, Type::Int32),
957  "");
958  const auto key_qw_count = get_slot_off_quad(query_mem_desc_);
959  const auto this_targets_ptr = ir_reduce_one_entry_idx->add<Cast>(
960  Cast::CastOp::BitCast, this_targets_ptr_i64, Type::Int8Ptr, "this_targets_ptr");
961  const auto key_byte_count = key_qw_count * sizeof(int64_t);
962  const auto key_byte_count_lv =
963  ir_reduce_one_entry_idx->addConstant<ConstantInt>(key_byte_count, Type::Int32);
964  const auto that_targets_ptr = ir_reduce_one_entry_idx->add<GetElementPtr>(
965  that_row_ptr, key_byte_count_lv, "that_targets_ptr");
966  const auto reduce_rc = ir_reduce_one_entry_idx->add<Call>(
967  reduction_code.ir_reduce_one_entry.get(),
968  std::vector<const Value*>{this_targets_ptr,
969  that_targets_ptr,
970  this_qmd_handle,
971  that_qmd_handle,
972  serialized_varlen_buffer_arg},
973  "");
974  ir_reduce_one_entry_idx->add<Ret>(reduce_rc);
975 }
976 
977 namespace {
978 
979 void generate_loop_body(For* for_loop,
980  Function* ir_reduce_loop,
981  Function* ir_reduce_one_entry_idx,
982  Value* this_buff,
983  Value* that_buff,
984  Value* start_index,
985  Value* that_entry_count,
986  Value* this_qmd_handle,
987  Value* that_qmd_handle,
988  Value* serialized_varlen_buffer) {
989  const auto that_entry_idx = for_loop->add<BinaryOperator>(
990  BinaryOperator::BinaryOp::Add, for_loop->iter(), start_index, "that_entry_idx");
991  const auto sample_seed =
992  for_loop->add<Cast>(Cast::CastOp::SExt, that_entry_idx, Type::Int64, "");
994  const auto checker_rt_name =
995  g_enable_dynamic_watchdog ? "check_watchdog_rt" : "check_interrupt_rt";
996  const auto error_code = g_enable_dynamic_watchdog ? WATCHDOG_ERROR : INTERRUPT_ERROR;
997  const auto checker_triggered = for_loop->add<ExternalCall>(
998  checker_rt_name, Type::Int8, std::vector<const Value*>{sample_seed}, "");
999  const auto interrupt_triggered_bool =
1000  for_loop->add<ICmp>(ICmp::Predicate::NE,
1001  checker_triggered,
1002  ir_reduce_loop->addConstant<ConstantInt>(0, Type::Int8),
1003  "");
1004  for_loop->add<ReturnEarly>(
1005  interrupt_triggered_bool,
1006  ir_reduce_loop->addConstant<ConstantInt>(error_code, Type::Int32),
1007  "");
1008  }
1009  const auto reduce_rc =
1010  for_loop->add<Call>(ir_reduce_one_entry_idx,
1011  std::vector<const Value*>{this_buff,
1012  that_buff,
1013  that_entry_idx,
1014  that_entry_count,
1015  this_qmd_handle,
1016  that_qmd_handle,
1017  serialized_varlen_buffer},
1018  "");
1019 
1020  auto reduce_rc_bool =
1021  for_loop->add<ICmp>(ICmp::Predicate::NE,
1022  reduce_rc,
1023  ir_reduce_loop->addConstant<ConstantInt>(0, Type::Int32),
1024  "");
1025  for_loop->add<ReturnEarly>(reduce_rc_bool, reduce_rc, "");
1026 }
1027 
1028 } // namespace
1029 
1030 void ResultSetReductionJIT::reduceLoop(const ReductionCode& reduction_code) const {
1031  auto ir_reduce_loop = reduction_code.ir_reduce_loop.get();
1032  const auto this_buff_arg = ir_reduce_loop->arg(0);
1033  const auto that_buff_arg = ir_reduce_loop->arg(1);
1034  const auto start_index_arg = ir_reduce_loop->arg(2);
1035  const auto end_index_arg = ir_reduce_loop->arg(3);
1036  const auto that_entry_count_arg = ir_reduce_loop->arg(4);
1037  const auto this_qmd_handle_arg = ir_reduce_loop->arg(5);
1038  const auto that_qmd_handle_arg = ir_reduce_loop->arg(6);
1039  const auto serialized_varlen_buffer_arg = ir_reduce_loop->arg(7);
1040  For* for_loop =
1041  static_cast<For*>(ir_reduce_loop->add<For>(start_index_arg, end_index_arg, ""));
1042  generate_loop_body(for_loop,
1043  ir_reduce_loop,
1044  reduction_code.ir_reduce_one_entry_idx.get(),
1045  this_buff_arg,
1046  that_buff_arg,
1047  start_index_arg,
1048  that_entry_count_arg,
1049  this_qmd_handle_arg,
1050  that_qmd_handle_arg,
1051  serialized_varlen_buffer_arg);
1052  ir_reduce_loop->add<Ret>(ir_reduce_loop->addConstant<ConstantInt>(0, Type::Int32));
1053 }
1054 
1056  Value* this_ptr2,
1057  Value* that_ptr1,
1058  Value* that_ptr2,
1059  const TargetInfo& target_info,
1060  const size_t target_logical_idx,
1061  const size_t target_slot_idx,
1062  const size_t init_agg_val_idx,
1063  const size_t first_slot_idx_for_target,
1064  Function* ir_reduce_one_entry) const {
1066  if (query_mem_desc_.getTargetGroupbyIndex(target_logical_idx) >= 0) {
1067  return;
1068  }
1069  }
1070  const bool float_argument_input = takes_float_argument(target_info);
1071  const auto chosen_bytes = result_set::get_width_for_slot(
1072  target_slot_idx, float_argument_input, query_mem_desc_);
1073  CHECK_LT(init_agg_val_idx, target_init_vals_.size());
1074  auto init_val = target_init_vals_[init_agg_val_idx];
1075  if (target_info.is_agg &&
1076  (target_info.agg_kind != kSINGLE_VALUE && target_info.agg_kind != kSAMPLE)) {
1077  reduceOneAggregateSlot(this_ptr1,
1078  this_ptr2,
1079  that_ptr1,
1080  that_ptr2,
1081  target_info,
1082  target_logical_idx,
1083  target_slot_idx,
1084  init_val,
1085  chosen_bytes,
1086  ir_reduce_one_entry);
1087  } else if (target_info.agg_kind == kSINGLE_VALUE) {
1088  const auto checked_rc = emit_checked_write_projection(
1089  this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1090 
1091  auto checked_rc_bool = ir_reduce_one_entry->add<ICmp>(
1093  checked_rc,
1094  ir_reduce_one_entry->addConstant<ConstantInt>(0, Type::Int32),
1095  "");
1096 
1097  ir_reduce_one_entry->add<ReturnEarly>(checked_rc_bool, checked_rc, "");
1098 
1099  } else {
1101  this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1102  if (target_info.agg_kind == kSAMPLE && target_info.sql_type.is_varlen()) {
1103  CHECK(this_ptr2 && that_ptr2);
1104  size_t length_to_elems{0};
1105  if (target_info.sql_type.is_geometry()) {
1106  // TODO: Assumes hard-coded sizes for geometry targets
1107  length_to_elems = target_slot_idx == first_slot_idx_for_target ? 1 : 4;
1108  } else {
1109  const auto& elem_ti = target_info.sql_type.get_elem_type();
1110  length_to_elems = target_info.sql_type.is_string() ? 1 : elem_ti.get_size();
1111  }
1112  const auto serialized_varlen_buffer_arg = ir_reduce_one_entry->arg(4);
1113  ir_reduce_one_entry->add<ExternalCall>(
1114  "serialized_varlen_buffer_sample",
1115  Type::Void,
1116  std::vector<const Value*>{
1117  serialized_varlen_buffer_arg,
1118  this_ptr1,
1119  this_ptr2,
1120  that_ptr1,
1121  that_ptr2,
1122  ir_reduce_one_entry->addConstant<ConstantInt>(init_val, Type::Int64),
1123  ir_reduce_one_entry->addConstant<ConstantInt>(length_to_elems,
1124  Type::Int64)},
1125  "");
1126  }
1127  }
1128 }
1129 
1131  Value* this_ptr2,
1132  Value* that_ptr1,
1133  Value* that_ptr2,
1134  const TargetInfo& target_info,
1135  const size_t target_logical_idx,
1136  const size_t target_slot_idx,
1137  const int64_t init_val,
1138  const int8_t chosen_bytes,
1139  Function* ir_reduce_one_entry) const {
1140  auto agg_kind = target_info.agg_kind;
1141  switch (agg_kind) {
1142  case kCOUNT:
1143  case kCOUNT_IF:
1145  if (is_distinct_target(target_info)) {
1146  CHECK_EQ(static_cast<size_t>(chosen_bytes), sizeof(int64_t));
1148  this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1149  break;
1150  }
1151  CHECK_EQ(int64_t(0), init_val);
1152  emit_aggregate_one_count(this_ptr1, that_ptr1, chosen_bytes, ir_reduce_one_entry);
1153  break;
1154  case kAPPROX_QUANTILE:
1155  CHECK_EQ(sizeof(int64_t), static_cast<size_t>(chosen_bytes));
1157  this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1158  break;
1159  case kMODE:
1160  reduceOneModeSlot(this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1161  break;
1162  case kAVG:
1163  // Ignore float argument compaction for count component for fear of its overflow
1164  emit_aggregate_one_count(this_ptr2,
1165  that_ptr2,
1166  query_mem_desc_.getPaddedSlotWidthBytes(target_slot_idx),
1167  ir_reduce_one_entry);
1168  agg_kind = kSUM;
1169  // fall thru
1170  case kSUM:
1171  case kMIN:
1172  case kMAX:
1173  // for conditional aggregation, we already "conditionally" aggregate value for each
1174  // resultset, so reduction just can use the non-conditional aggregation logic
1175  case kSUM_IF:
1177  this_ptr1,
1178  that_ptr1,
1179  init_val,
1180  chosen_bytes,
1181  target_info,
1182  ir_reduce_one_entry);
1183  break;
1184  default:
1185  UNREACHABLE() << "Invalid aggregate type: " << agg_kind;
1186  }
1187 }
1188 
1190  Value* this_ptr1,
1191  Value* that_ptr1,
1192  const size_t target_logical_idx,
1193  Function* ir_reduce_one_entry) const {
1195  const auto old_set_handle = emit_load_i64(this_ptr1, ir_reduce_one_entry);
1196  const auto new_set_handle = emit_load_i64(that_ptr1, ir_reduce_one_entry);
1197  const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1198  const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1199  ir_reduce_one_entry->add<ExternalCall>(
1200  "count_distinct_set_union_jit_rt",
1201  Type::Void,
1202  std::vector<const Value*>{
1203  new_set_handle,
1204  old_set_handle,
1205  that_qmd_arg,
1206  this_qmd_arg,
1207  ir_reduce_one_entry->addConstant<ConstantInt>(target_logical_idx, Type::Int64)},
1208  "");
1209 }
1210 
1212  Value* this_ptr1,
1213  Value* that_ptr1,
1214  const size_t target_logical_idx,
1215  Function* ir_reduce_one_entry) const {
1217  const auto old_set_handle = emit_load_i64(this_ptr1, ir_reduce_one_entry);
1218  const auto new_set_handle = emit_load_i64(that_ptr1, ir_reduce_one_entry);
1219  const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1220  const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1221  ir_reduce_one_entry->add<ExternalCall>(
1222  "approx_quantile_jit_rt",
1223  Type::Void,
1224  std::vector<const Value*>{
1225  new_set_handle,
1226  old_set_handle,
1227  that_qmd_arg,
1228  this_qmd_arg,
1229  ir_reduce_one_entry->addConstant<ConstantInt>(target_logical_idx, Type::Int64)},
1230  "");
1231 }
1232 
1234  Value* that_ptr1,
1235  const size_t target_logical_idx,
1236  Function* ir_reduce_one_entry) const {
1238  const auto old_set_handle = emit_load_i64(this_ptr1, ir_reduce_one_entry);
1239  const auto new_set_handle = emit_load_i64(that_ptr1, ir_reduce_one_entry);
1240  const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1241  const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1242  ir_reduce_one_entry->add<ExternalCall>(
1243  "mode_jit_rt",
1244  Type::Void,
1245  std::vector<const Value*>{
1246  new_set_handle,
1247  old_set_handle,
1248  that_qmd_arg,
1249  this_qmd_arg,
1250  ir_reduce_one_entry->addConstant<ConstantInt>(target_logical_idx, Type::Int64)},
1251  "");
1252 }
1253 
1255  ReductionCode& reduction_code,
1256  const llvm::Function* ir_is_empty,
1257  const llvm::Function* ir_reduce_one_entry,
1258  const llvm::Function* ir_reduce_one_entry_idx,
1259  const CodeCacheKey& key) const {
1260  CompilationOptions co{
1262  VLOG(3) << "Reduction Loop:\n"
1263  << serialize_llvm_object(reduction_code.llvm_reduce_loop);
1264  VLOG(3) << "Reduction Is Empty Func:\n" << serialize_llvm_object(ir_is_empty);
1265  VLOG(3) << "Reduction One Entry Func:\n" << serialize_llvm_object(ir_reduce_one_entry);
1266  VLOG(3) << "Reduction One Entry Idx Func:\n"
1267  << serialize_llvm_object(ir_reduce_one_entry_idx);
1268 #ifdef NDEBUG
1269  LOG(IR) << "Reduction Loop:\n"
1270  << serialize_llvm_object(reduction_code.llvm_reduce_loop);
1271  LOG(IR) << "Reduction Is Empty Func:\n" << serialize_llvm_object(ir_is_empty);
1272  LOG(IR) << "Reduction One Entry Func:\n" << serialize_llvm_object(ir_reduce_one_entry);
1273  LOG(IR) << "Reduction One Entry Idx Func:\n"
1274  << serialize_llvm_object(ir_reduce_one_entry_idx);
1275 #else
1276  LOG(IR) << serialize_llvm_object(reduction_code.module);
1277 #endif
1279  reduction_code.llvm_reduce_loop, {reduction_code.llvm_reduce_loop}, co);
1280  auto cpu_compilation_context = std::make_shared<CpuCompilationContext>(std::move(ee));
1281  cpu_compilation_context->setFunctionPointer(reduction_code.llvm_reduce_loop);
1282  reduction_code.func_ptr =
1283  reinterpret_cast<ReductionCode::FuncPtr>(cpu_compilation_context->func());
1284  CHECK(reduction_code.llvm_reduce_loop->getParent() == reduction_code.module);
1285  auto executor = Executor::getExecutor(executor_id_);
1286  QueryEngine::getInstance()->s_code_accessor->swap(key,
1287  std::move(cpu_compilation_context));
1288 }
1289 
1290 namespace {
1291 
1292 std::string target_info_key(const TargetInfo& target_info) {
1293  return std::to_string(target_info.is_agg) + "\n" +
1294  std::to_string(target_info.agg_kind) + "\n" +
1295  target_info.sql_type.get_type_name() + "\n" +
1296  std::to_string(target_info.sql_type.get_notnull()) + "\n" +
1297  target_info.agg_arg_type.get_type_name() + "\n" +
1298  std::to_string(target_info.agg_arg_type.get_notnull()) + "\n" +
1299  std::to_string(target_info.skip_null_val) + "\n" +
1300  std::to_string(target_info.is_distinct);
1301 }
1302 
1303 } // namespace
1304 
1305 std::string ResultSetReductionJIT::cacheKey() const {
1306  std::vector<std::string> target_init_vals_strings;
1308  target_init_vals_.end(),
1309  std::back_inserter(target_init_vals_strings),
1310  [](const int64_t v) { return std::to_string(v); });
1311  const auto target_init_vals_key =
1312  boost::algorithm::join(target_init_vals_strings, ", ");
1313  std::vector<std::string> targets_strings;
1315  targets_.begin(),
1316  targets_.end(),
1317  std::back_inserter(targets_strings),
1318  [](const TargetInfo& target_info) { return target_info_key(target_info); });
1319  const auto targets_key = boost::algorithm::join(targets_strings, ", ");
1320  return query_mem_desc_.reductionKey() + "\n" + target_init_vals_key + "\n" +
1321  targets_key;
1322 }
1323 
1325  const auto hash_type = query_mem_desc_.getQueryDescriptionType();
1326  auto reduction_code = setup_functions_ir(hash_type);
1328  isEmpty(reduction_code);
1329  reduceOneEntryNoCollisions(reduction_code);
1330  reduceOneEntryNoCollisionsIdx(reduction_code);
1331  reduceLoop(reduction_code);
1332  auto executor = Executor::getExecutor(executor_id_);
1333  auto cgen_state_ = std::unique_ptr<CgenState>(new CgenState({}, false, executor.get()));
1334  auto cgen_state = reduction_code.cgen_state = cgen_state_.get();
1335  // CHECK(executor->thread_id_ == logger::thread_id()); // do we need compilation mutex?
1336  cgen_state->set_module_shallow_copy(executor->get_rt_module());
1337  reduction_code.module = cgen_state->module_;
1338 
1339  AUTOMATIC_IR_METADATA(cgen_state);
1340  auto ir_is_empty = create_llvm_function(reduction_code.ir_is_empty.get(), cgen_state);
1341  auto ir_reduce_one_entry =
1342  create_llvm_function(reduction_code.ir_reduce_one_entry.get(), cgen_state);
1343  auto ir_reduce_one_entry_idx =
1344  create_llvm_function(reduction_code.ir_reduce_one_entry_idx.get(), cgen_state);
1345  auto ir_reduce_loop =
1346  create_llvm_function(reduction_code.ir_reduce_loop.get(), cgen_state);
1347  std::unordered_map<const Function*, llvm::Function*> f;
1348  f.emplace(reduction_code.ir_is_empty.get(), ir_is_empty);
1349  f.emplace(reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry);
1350  f.emplace(reduction_code.ir_reduce_one_entry_idx.get(), ir_reduce_one_entry_idx);
1351  f.emplace(reduction_code.ir_reduce_loop.get(), ir_reduce_loop);
1352  translate_function(reduction_code.ir_is_empty.get(), ir_is_empty, reduction_code, f);
1354  reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code, f);
1355  translate_function(reduction_code.ir_reduce_one_entry_idx.get(),
1356  ir_reduce_one_entry_idx,
1357  reduction_code,
1358  f);
1360  reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code, f);
1361  reduction_code.llvm_reduce_loop = ir_reduce_loop;
1362  reduction_code.cgen_state = nullptr;
1363  return reduction_code;
1364 }
GroupValueInfo get_group_value_reduction(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const size_t key_width, const QueryMemoryDescriptor &query_mem_desc, const int64_t *that_buff_i64, const size_t that_entry_idx, const size_t that_entry_count, const uint32_t row_size_quad)
std::string to_lower(const std::string &str)
SQLAgg
Definition: sqldefs.h:73
#define CHECK_EQ(x, y)
Definition: Logger.h:301
CgenState * cgen_state
void reduceOneSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const size_t init_agg_val_idx, const size_t first_slot_idx_for_target, Function *ir_reduce_one_entry) const
void reduce(AggMode &&rhs)
Definition: AggMode.h:47
bool is_aggregate_query(const QueryDescriptionType hash_type)
void count_distinct_set_union(const int64_t new_set_handle, const int64_t old_set_handle, const CountDistinctDescriptor &new_count_distinct_desc, const CountDistinctDescriptor &old_count_distinct_desc)
__device__ bool dynamic_watchdog()
#define EMPTY_KEY_64
const std::string & label() const
RUNTIME_EXPORT uint8_t check_watchdog_rt(const size_t sample_seed)
void reduceOneEntryNoCollisions(const ReductionCode &reduction_code) const
int64_t getTargetGroupbyIndex(const size_t target_idx) const
void varlen_buffer_sample(int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val)
std::unique_ptr< Function > ir_reduce_loop
Value * emit_read_int_from_buff(Value *ptr, const int8_t compact_sz, Function *function)
void reduceOneEntryBaselineIdx(const ReductionCode &reduction_code) const
SQLTypeInfo sql_type
Definition: TargetInfo.h:52
#define LOG(tag)
Definition: Logger.h:285
void mark_function_always_inline(llvm::Function *func)
Calculate approximate median and general quantiles, based on &quot;Computing Extremely Accurate Quantiles ...
void reduceLoop(const ReductionCode &reduction_code) const
bool is_varlen() const
Definition: sqltypes.h:620
std::string join(T const &container, std::string const &delim)
llvm::Function * llvm_reduce_loop
#define UNREACHABLE()
Definition: Logger.h:337
void reduceOneEntryNoCollisionsIdx(const ReductionCode &reduction_code) const
#define CHECK_GE(x, y)
Definition: Logger.h:306
std::vector< std::string > CodeCacheKey
Definition: CodeCache.h:25
size_t get_slot_off_quad(const QueryMemoryDescriptor &query_mem_desc)
std::string cacheKey() const
size_t getEffectiveKeyWidth() const
std::unique_ptr< Function > ir_reduce_one_entry
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:80
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co)
const std::vector< int64_t > target_init_vals_
void reduceOneAggregateSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const int64_t init_val, const int8_t chosen_bytes, Function *ir_reduce_one_entry) const
bool takes_float_argument(const TargetInfo &target_info)
Definition: TargetInfo.h:102
bool g_enable_non_kernel_time_query_interrupt
Definition: Execute.cpp:126
Value * add(Args &&...args)
bool skip_null_val
Definition: TargetInfo.h:54
const Value * emit_checked_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
std::unique_ptr< Function > setup_reduce_one_entry_idx(ReductionCode *reduction_code)
int32_t(*)(int8_t *this_buff, const int8_t *that_buff, const int32_t start_entry_index, const int32_t end_entry_index, const int32_t that_entry_count, const void *this_qmd, const void *that_qmd, const void *serialized_varlen_buffer) FuncPtr
const QueryMemoryDescriptor query_mem_desc_
std::unique_ptr< Function > ir_is_empty
constexpr double f
Definition: Utm.h:31
std::string to_string(char const *&&v)
SQLTypeInfo agg_arg_type
Definition: TargetInfo.h:53
void translate_function(const Function *function, llvm::Function *llvm_function, const ReductionCode &reduction_code, const std::unordered_map< const Function *, llvm::Function * > &f)
void emit_aggregate_one_value(const std::string &agg_kind, Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
RUNTIME_EXPORT void serialized_varlen_buffer_sample(const void *serialized_varlen_buffer_handle, int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val, const int64_t length_to_elems)
Value * emit_load_i32(Value *ptr, Function *function)
Definition: sqldefs.h:75
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
const SQLTypeInfo get_compact_type(const TargetInfo &target)
__device__ bool check_interrupt()
int8_t get_width_for_slot(const size_t target_slot_idx, const bool float_argument_input, const QueryMemoryDescriptor &query_mem_desc)
llvm::Module * module_
Definition: CgenState.h:366
llvm::LLVMContext & context_
Definition: CgenState.h:375
size_t get_byteoff_of_slot(const size_t slot_idx, const QueryMemoryDescriptor &query_mem_desc)
bool is_agg
Definition: TargetInfo.h:50
size_t advance_slot(const size_t j, const TargetInfo &target_info, const bool separate_varlen_storage)
void reduceOneCountDistinctSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
uint8_t check_interrupt_rt(const size_t sample_seed)
size_t getGroupbyColCount() const
size_t targetGroupbyIndicesSize() const
void generate_loop_body(For *for_loop, Function *ir_reduce_loop, Function *ir_reduce_one_entry_idx, Value *this_buff, Value *that_buff, Value *start_index, Value *that_entry_count, Value *this_qmd_handle, Value *that_qmd_handle, Value *serialized_varlen_buffer)
void emit_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
void emit_aggregate_one_nullable_value(SQLAgg const sql_agg, Value *val_ptr, Value *other_ptr, const int64_t init_val, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
Definition: sqldefs.h:77
ReductionCode codegen() const override
bool is_distinct_target(const TargetInfo &target_info)
Definition: TargetInfo.h:98
std::string toString(const ExecutorDeviceType &device_type)
std::string target_info_key(const TargetInfo &target_info)
OUTPUT transform(INPUT const &input, FUNC const &func)
Definition: misc.h:320
std::unique_ptr< Function > ir_reduce_one_entry_idx
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
ReductionCode setup_functions_ir(const QueryDescriptionType hash_type)
DEVICE void allocate()
Definition: quantile.h:601
#define AUTOMATIC_IR_METADATA(CGENSTATE)
std::unique_ptr< Function > setup_is_empty_entry(ReductionCode *reduction_code)
SQLAgg agg_kind
Definition: TargetInfo.h:51
size_t getCountDistinctDescriptorsSize() const
void reduceOneEntryTargetsNoCollisions(Function *ir_reduce_one_entry, Value *this_targets_start_ptr, Value *that_targets_start_ptr) const
QueryDescriptionType getQueryDescriptionType() const
#define AUTOMATIC_IR_METADATA_DONE()
#define UNLIKELY(x)
Definition: likely.h:25
void set_module_shallow_copy(const std::unique_ptr< llvm::Module > &module, bool always_clone=false)
Definition: CgenState.cpp:381
#define RUNTIME_EXPORT
llvm::Type * llvm_type(const Type type, llvm::LLVMContext &ctx)
virtual ReductionCode codegen() const
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
RUNTIME_EXPORT void mode_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
llvm::Module * module
#define CHECK_LT(x, y)
Definition: Logger.h:303
std::string serialize_llvm_object(const T *llvm_obj)
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
RUNTIME_EXPORT void count_distinct_set_union_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
void finalizeReductionCode(ReductionCode &reduction_code, const llvm::Function *ir_is_empty, const llvm::Function *ir_reduce_one_entry, const llvm::Function *ir_reduce_one_entry_idx, const CodeCacheKey &key) const
Definition: sqldefs.h:78
std::unique_ptr< Function > create_function(const std::string name, const std::vector< Function::NamedArg > &arg_types, const Type ret_type, const bool always_inline)
std::string get_type_name() const
Definition: sqltypes.h:507
const Value * iter() const
RUNTIME_EXPORT void approx_quantile_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
RUNTIME_EXPORT void get_group_value_reduction_rt(int8_t *groups_buffer, const int8_t *key, const uint32_t key_count, const void *this_qmd_handle, const int8_t *that_buff, const uint32_t that_entry_idx, const uint32_t that_entry_count, const uint32_t row_size_bytes, int64_t **buff_out, uint8_t *empty)
std::unique_ptr< Function > setup_reduce_one_entry(ReductionCode *reduction_code, const QueryDescriptionType hash_type)
void isEmpty(const ReductionCode &reduction_code) const
Value * emit_load_i64(Value *ptr, Function *function)
const ColSlotContext & getColSlotContext() const
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:592
#define EMPTY_KEY_32
void reduceOneEntryBaseline(const ReductionCode &reduction_code) const
QueryDescriptionType
Definition: Types.h:29
static std::shared_ptr< QueryEngine > getInstance()
Definition: QueryEngine.h:81
Value * emit_load(Value *ptr, Type ptr_type, Function *function)
bool is_string() const
Definition: sqltypes.h:580
llvm::Function * create_llvm_function(const Function *function, CgenState *cgen_state)
ResultSetReductionJIT(const QueryMemoryDescriptor &query_mem_desc, const std::vector< TargetInfo > &targets, const std::vector< int64_t > &target_init_vals, const size_t executor_id)
string name
Definition: setup.in.py:72
bool is_distinct
Definition: TargetInfo.h:55
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:388
void emit_aggregate_one_count(Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, Function *ir_reduce_one_entry)
Definition: sqldefs.h:76
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:963
Definition: sqldefs.h:74
void reduceOneApproxQuantileSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
std::unique_ptr< Function > setup_reduce_loop(ReductionCode *reduction_code)
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
Definition: sqldefs.h:83
SQLAgg get_non_conditional_agg_type(SQLAgg const agg_type)
Definition: sqldefs.h:247
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
#define VLOG(n)
Definition: Logger.h:387
std::string reductionKey() const
const Executor * getExecutor() const
void reduceOneModeSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
int32_t getTargetIdxForKey() const
const std::vector< TargetInfo > targets_