OmniSciDB  0bd2ec9cf4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
TargetExprBuilder.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "TargetExprBuilder.h"
24 
25 #include "CodeGenerator.h"
26 #include "Execute.h"
27 #include "GroupByAndAggregate.h"
28 #include "MaxwellCodegenPatch.h"
30 #include "Shared/Logger.h"
31 
32 #define LL_CONTEXT executor->cgen_state_->context_
33 #define LL_BUILDER executor->cgen_state_->ir_builder_
34 #define LL_BOOL(v) executor->ll_bool(v)
35 #define LL_INT(v) executor->cgen_state_->llInt(v)
36 #define LL_FP(v) executor->cgen_state_->llFp(v)
37 #define ROW_FUNC executor->cgen_state_->row_func_
38 
39 namespace {
40 
41 std::vector<std::string> agg_fn_base_names(const TargetInfo& target_info) {
42  const auto& chosen_type = get_compact_type(target_info);
43  if (!target_info.is_agg || target_info.agg_kind == kSAMPLE) {
44  if (chosen_type.is_geometry()) {
45  return std::vector<std::string>(2 * chosen_type.get_physical_coord_cols(),
46  "agg_id");
47  }
48  if (chosen_type.is_varlen()) {
49  return {"agg_id", "agg_id"};
50  }
51  return {"agg_id"};
52  }
53  switch (target_info.agg_kind) {
54  case kAVG:
55  return {"agg_sum", "agg_count"};
56  case kCOUNT:
57  return {target_info.is_distinct ? "agg_count_distinct" : "agg_count"};
58  case kMAX:
59  return {"agg_max"};
60  case kMIN:
61  return {"agg_min"};
62  case kSUM:
63  return {"agg_sum"};
65  return {"agg_approximate_count_distinct"};
66  case kSINGLE_VALUE:
67  return {"checked_single_agg_id"};
68  case kSAMPLE:
69  return {"agg_id"};
70  default:
71  UNREACHABLE() << "Unrecognized agg kind: " << std::to_string(target_info.agg_kind);
72  }
73  return {};
74 }
75 
77  return query_mem_desc.getQueryDescriptionType() == QueryDescriptionType::Projection &&
78  query_mem_desc.didOutputColumnar();
79 }
80 
81 } // namespace
82 
84  GroupByAndAggregate* group_by_and_agg,
85  Executor* executor,
87  const CompilationOptions& co,
88  const std::tuple<llvm::Value*, llvm::Value*>& agg_out_ptr_w_idx_in,
89  const std::vector<llvm::Value*>& agg_out_vec,
90  llvm::Value* output_buffer_byte_stream,
91  llvm::Value* out_row_idx,
92  GroupByAndAggregate::DiamondCodegen& diamond_codegen,
93  GroupByAndAggregate::DiamondCodegen* sample_cfg) const {
94  CHECK(group_by_and_agg);
95  CHECK(executor);
96 
97  auto agg_out_ptr_w_idx = agg_out_ptr_w_idx_in;
98  const auto arg_expr = agg_arg(target_expr);
99 
100  const auto agg_fn_names = agg_fn_base_names(target_info);
101  const auto window_func = dynamic_cast<const Analyzer::WindowFunction*>(target_expr);
103  auto target_lvs =
104  window_func
105  ? std::vector<llvm::Value*>{executor->codegenWindowFunction(target_idx, co)}
106  : group_by_and_agg->codegenAggArg(target_expr, co);
107  const auto window_row_ptr = window_func
108  ? group_by_and_agg->codegenWindowRowPointer(
109  window_func, query_mem_desc, co, diamond_codegen)
110  : nullptr;
111  if (window_row_ptr) {
112  agg_out_ptr_w_idx =
113  std::make_tuple(window_row_ptr, std::get<1>(agg_out_ptr_w_idx_in));
114  if (window_function_is_aggregate(window_func->getKind())) {
115  out_row_idx = window_row_ptr;
116  }
117  }
118 
119  llvm::Value* str_target_lv{nullptr};
120  const bool target_has_geo = target_info.is_agg ? target_info.agg_arg_type.is_geometry()
122  if (target_lvs.size() == 3 && !target_has_geo) {
123  // none encoding string, pop the packed pointer + length since
124  // it's only useful for IS NULL checks and assumed to be only
125  // two components (pointer and length) for the purpose of projection
126  str_target_lv = target_lvs.front();
127  target_lvs.erase(target_lvs.begin());
128  }
130  // Geo cols are expanded to the physical coord cols. Each physical coord col is an
131  // array. Ensure that the target values generated match the number of agg
132  // functions before continuing
133  if (target_lvs.size() < agg_fn_names.size()) {
134  CHECK_EQ(target_lvs.size(), agg_fn_names.size() / 2);
135  std::vector<llvm::Value*> new_target_lvs;
136  new_target_lvs.reserve(agg_fn_names.size());
137  for (const auto& target_lv : target_lvs) {
138  new_target_lvs.push_back(target_lv);
139  new_target_lvs.push_back(target_lv);
140  }
141  target_lvs = new_target_lvs;
142  }
143  }
144  if (target_lvs.size() < agg_fn_names.size()) {
145  CHECK_EQ(size_t(1), target_lvs.size());
146  CHECK_EQ(size_t(2), agg_fn_names.size());
147  for (size_t i = 1; i < agg_fn_names.size(); ++i) {
148  target_lvs.push_back(target_lvs.front());
149  }
150  } else {
151  if (target_has_geo) {
152  if (!target_info.is_agg) {
153  CHECK_EQ(static_cast<size_t>(2 * target_info.sql_type.get_physical_coord_cols()),
154  target_lvs.size());
155  CHECK_EQ(agg_fn_names.size(), target_lvs.size());
156  }
157  } else {
158  CHECK(str_target_lv || (agg_fn_names.size() == target_lvs.size()));
159  CHECK(target_lvs.size() == 1 || target_lvs.size() == 2);
160  }
161  }
162 
163  int32_t slot_index = base_slot_index;
164  CHECK_GE(slot_index, 0);
165  CHECK(is_group_by || static_cast<size_t>(slot_index) < agg_out_vec.size());
166 
167  uint32_t col_off{0};
168  const bool is_simple_count =
170  if (co.device_type_ == ExecutorDeviceType::GPU && query_mem_desc.threadsShareMemory() &&
171  is_simple_count && (!arg_expr || arg_expr->get_type_info().get_notnull())) {
172  CHECK_EQ(size_t(1), agg_fn_names.size());
173  const auto chosen_bytes = query_mem_desc.getPaddedSlotWidthBytes(slot_index);
174  llvm::Value* agg_col_ptr{nullptr};
175  if (is_group_by) {
176  if (query_mem_desc.didOutputColumnar()) {
177  col_off = query_mem_desc.getColOffInBytes(slot_index);
178  CHECK_EQ(size_t(0), col_off % chosen_bytes);
179  col_off /= chosen_bytes;
180  CHECK(std::get<1>(agg_out_ptr_w_idx));
181  auto offset =
182  LL_BUILDER.CreateAdd(std::get<1>(agg_out_ptr_w_idx), LL_INT(col_off));
183  agg_col_ptr = LL_BUILDER.CreateGEP(
184  LL_BUILDER.CreateBitCast(
185  std::get<0>(agg_out_ptr_w_idx),
186  llvm::PointerType::get(get_int_type((chosen_bytes << 3), LL_CONTEXT), 0)),
187  offset);
188  } else {
189  col_off = query_mem_desc.getColOnlyOffInBytes(slot_index);
190  CHECK_EQ(size_t(0), col_off % chosen_bytes);
191  col_off /= chosen_bytes;
192  agg_col_ptr = LL_BUILDER.CreateGEP(
193  LL_BUILDER.CreateBitCast(
194  std::get<0>(agg_out_ptr_w_idx),
195  llvm::PointerType::get(get_int_type((chosen_bytes << 3), LL_CONTEXT), 0)),
196  LL_INT(col_off));
197  }
198  }
199 
200  if (chosen_bytes != sizeof(int32_t)) {
201  CHECK_EQ(8, chosen_bytes);
202  if (g_bigint_count) {
203  const auto acc_i64 = LL_BUILDER.CreateBitCast(
204  is_group_by ? agg_col_ptr : agg_out_vec[slot_index],
205  llvm::PointerType::get(get_int_type(64, LL_CONTEXT), 0));
206  LL_BUILDER.CreateAtomicRMW(llvm::AtomicRMWInst::Add,
207  acc_i64,
208  LL_INT(int64_t(1)),
209  llvm::AtomicOrdering::Monotonic);
210  } else {
211  const auto acc_i32 = LL_BUILDER.CreateBitCast(
212  is_group_by ? agg_col_ptr : agg_out_vec[slot_index],
213  llvm::PointerType::get(get_int_type(32, LL_CONTEXT), 0));
214  LL_BUILDER.CreateAtomicRMW(llvm::AtomicRMWInst::Add,
215  acc_i32,
216  LL_INT(1),
217  llvm::AtomicOrdering::Monotonic);
218  }
219  } else {
220  const auto acc_i32 = (is_group_by ? agg_col_ptr : agg_out_vec[slot_index]);
221  if (query_mem_desc.getGpuMemSharing() ==
223  // Atomic operation on address space level 3 (Shared):
224  const auto shared_acc_i32 = LL_BUILDER.CreatePointerCast(
225  acc_i32, llvm::Type::getInt32PtrTy(LL_CONTEXT, 3));
226  LL_BUILDER.CreateAtomicRMW(llvm::AtomicRMWInst::Add,
227  shared_acc_i32,
228  LL_INT(1),
229  llvm::AtomicOrdering::Monotonic);
230  } else {
231  LL_BUILDER.CreateAtomicRMW(llvm::AtomicRMWInst::Add,
232  acc_i32,
233  LL_INT(1),
234  llvm::AtomicOrdering::Monotonic);
235  }
236  }
237  return;
238  }
239 
240  size_t target_lv_idx = 0;
241  const bool lazy_fetched{executor->plan_state_->isLazyFetchColumn(target_expr)};
242 
243  CodeGenerator code_generator(executor);
244  for (const auto& agg_base_name : agg_fn_names) {
245  if (target_info.is_distinct && arg_expr->get_type_info().is_array()) {
246  CHECK_EQ(static_cast<size_t>(query_mem_desc.getLogicalSlotWidthBytes(slot_index)),
247  sizeof(int64_t));
248  // TODO(miyu): check if buffer may be columnar here
249  CHECK(!query_mem_desc.didOutputColumnar());
250  const auto& elem_ti = arg_expr->get_type_info().get_elem_type();
251  if (is_group_by) {
252  col_off = query_mem_desc.getColOnlyOffInBytes(slot_index);
253  CHECK_EQ(size_t(0), col_off % sizeof(int64_t));
254  col_off /= sizeof(int64_t);
255  }
256  executor->cgen_state_->emitExternalCall(
257  "agg_count_distinct_array_" + numeric_type_name(elem_ti),
258  llvm::Type::getVoidTy(LL_CONTEXT),
259  {is_group_by
260  ? LL_BUILDER.CreateGEP(std::get<0>(agg_out_ptr_w_idx), LL_INT(col_off))
261  : agg_out_vec[slot_index],
262  target_lvs[target_lv_idx],
263  code_generator.posArg(arg_expr),
264  elem_ti.is_fp()
265  ? static_cast<llvm::Value*>(executor->cgen_state_->inlineFpNull(elem_ti))
266  : static_cast<llvm::Value*>(
267  executor->cgen_state_->inlineIntNull(elem_ti))});
268  ++slot_index;
269  ++target_lv_idx;
270  continue;
271  }
272 
273  llvm::Value* agg_col_ptr{nullptr};
274  const auto chosen_bytes =
275  static_cast<size_t>(query_mem_desc.getPaddedSlotWidthBytes(slot_index));
276  const auto& chosen_type = get_compact_type(target_info);
277  const auto& arg_type =
278  ((arg_expr && arg_expr->get_type_info().get_type() != kNULLT) &&
282  const bool is_fp_arg =
283  !lazy_fetched && arg_type.get_type() != kNULLT && arg_type.is_fp();
284  if (is_group_by) {
285  agg_col_ptr = group_by_and_agg->codegenAggColumnPtr(output_buffer_byte_stream,
286  out_row_idx,
287  agg_out_ptr_w_idx,
288  query_mem_desc,
289  chosen_bytes,
290  slot_index,
291  target_idx);
292  CHECK(agg_col_ptr);
293  agg_col_ptr->setName("agg_col_ptr");
294  }
295 
296  const bool float_argument_input = takes_float_argument(target_info);
297  const bool is_count_in_avg = target_info.agg_kind == kAVG && target_lv_idx == 1;
298  // The count component of an average should never be compacted.
299  const auto agg_chosen_bytes =
300  float_argument_input && !is_count_in_avg ? sizeof(float) : chosen_bytes;
301  if (float_argument_input) {
302  CHECK_GE(chosen_bytes, sizeof(float));
303  }
304 
305  auto target_lv = target_lvs[target_lv_idx];
306  const auto needs_unnest_double_patch = group_by_and_agg->needsUnnestDoublePatch(
307  target_lv, agg_base_name, query_mem_desc.threadsShareMemory(), co);
308  const auto need_skip_null = !needs_unnest_double_patch && target_info.skip_null_val;
309  if (!needs_unnest_double_patch) {
310  if (need_skip_null && !is_agg_domain_range_equivalent(target_info.agg_kind)) {
311  target_lv = group_by_and_agg->convertNullIfAny(arg_type, target_info, target_lv);
312  } else if (is_fp_arg) {
313  target_lv = executor->castToFP(target_lv);
314  }
315  if (!dynamic_cast<const Analyzer::AggExpr*>(target_expr) || arg_expr) {
316  target_lv =
317  executor->cgen_state_->castToTypeIn(target_lv, (agg_chosen_bytes << 3));
318  }
319  }
320 
321  std::vector<llvm::Value*> agg_args{
322  executor->castToIntPtrTyIn((is_group_by ? agg_col_ptr : agg_out_vec[slot_index]),
323  (agg_chosen_bytes << 3)),
324  (is_simple_count && !arg_expr)
325  ? (agg_chosen_bytes == sizeof(int32_t) ? LL_INT(int32_t(0))
326  : LL_INT(int64_t(0)))
327  : (is_simple_count && arg_expr && str_target_lv ? str_target_lv : target_lv)};
328  if (query_mem_desc.isLogicalSizedColumnsAllowed()) {
329  if (is_simple_count && arg_expr && str_target_lv) {
330  agg_args[1] =
331  agg_chosen_bytes == sizeof(int32_t) ? LL_INT(int32_t(0)) : LL_INT(int64_t(0));
332  }
333  }
334  std::string agg_fname{agg_base_name};
335  if (is_fp_arg) {
336  if (!lazy_fetched) {
337  if (agg_chosen_bytes == sizeof(float)) {
338  CHECK_EQ(arg_type.get_type(), kFLOAT);
339  agg_fname += "_float";
340  } else {
341  CHECK_EQ(agg_chosen_bytes, sizeof(double));
342  agg_fname += "_double";
343  }
344  }
345  } else if (agg_chosen_bytes == sizeof(int32_t)) {
346  agg_fname += "_int32";
347  } else if (agg_chosen_bytes == sizeof(int16_t) &&
348  query_mem_desc.didOutputColumnar()) {
349  agg_fname += "_int16";
350  } else if (agg_chosen_bytes == sizeof(int8_t) && query_mem_desc.didOutputColumnar()) {
351  agg_fname += "_int8";
352  }
353 
355  CHECK_EQ(agg_chosen_bytes, sizeof(int64_t));
356  CHECK(!chosen_type.is_fp());
357  group_by_and_agg->codegenCountDistinct(
358  target_idx, target_expr, agg_args, query_mem_desc, co.device_type_);
359  } else {
360  const auto& arg_ti = target_info.agg_arg_type;
361  if (need_skip_null && !arg_ti.is_geometry()) {
362  agg_fname += "_skip_val";
363  }
364 
366  (need_skip_null && !arg_ti.is_geometry())) {
367  llvm::Value* null_in_lv{nullptr};
368  if (arg_ti.is_fp()) {
369  null_in_lv =
370  static_cast<llvm::Value*>(executor->cgen_state_->inlineFpNull(arg_ti));
371  } else {
372  null_in_lv = static_cast<llvm::Value*>(executor->cgen_state_->inlineIntNull(
374  ? arg_ti
375  : target_info.sql_type));
376  }
377  CHECK(null_in_lv);
378  auto null_lv =
379  executor->cgen_state_->castToTypeIn(null_in_lv, (agg_chosen_bytes << 3));
380  agg_args.push_back(null_lv);
381  }
382  if (!target_info.is_distinct) {
384  query_mem_desc.threadsShareMemory()) {
385  agg_fname += "_shared";
386  if (needs_unnest_double_patch) {
387  agg_fname = patch_agg_fname(agg_fname);
388  }
389  }
390  auto agg_fname_call_ret_lv = group_by_and_agg->emitCall(agg_fname, agg_args);
391 
392  if (agg_fname.find("checked") != std::string::npos) {
393  group_by_and_agg->checkErrorCode(agg_fname_call_ret_lv);
394  }
395  }
396  }
397  if (window_func && window_function_requires_peer_handling(window_func)) {
398  const auto window_func_context =
400  const auto pending_outputs =
401  LL_INT(window_func_context->aggregateStatePendingOutputs());
402  executor->cgen_state_->emitExternalCall("add_window_pending_output",
403  llvm::Type::getVoidTy(LL_CONTEXT),
404  {agg_args.front(), pending_outputs});
405  const auto& window_func_ti = window_func->get_type_info();
406  std::string apply_window_pending_outputs_name = "apply_window_pending_outputs";
407  switch (window_func_ti.get_type()) {
408  case kFLOAT: {
409  apply_window_pending_outputs_name += "_float";
410  if (query_mem_desc.didOutputColumnar()) {
411  apply_window_pending_outputs_name += "_columnar";
412  }
413  break;
414  }
415  case kDOUBLE: {
416  apply_window_pending_outputs_name += "_double";
417  break;
418  }
419  default: {
420  apply_window_pending_outputs_name += "_int";
421  if (query_mem_desc.didOutputColumnar()) {
422  apply_window_pending_outputs_name +=
423  std::to_string(window_func_ti.get_size() * 8);
424  } else {
425  apply_window_pending_outputs_name += "64";
426  }
427  break;
428  }
429  }
430  const auto partition_end =
431  LL_INT(reinterpret_cast<int64_t>(window_func_context->partitionEnd()));
432  executor->cgen_state_->emitExternalCall(apply_window_pending_outputs_name,
433  llvm::Type::getVoidTy(LL_CONTEXT),
434  {pending_outputs,
435  target_lvs.front(),
436  partition_end,
437  code_generator.posArg(nullptr)});
438  }
439 
440  ++slot_index;
441  ++target_lv_idx;
442  }
443 }
444 
446  const Executor* executor,
447  const CompilationOptions& co) {
448  if (query_mem_desc.getPaddedSlotWidthBytes(slot_index_counter) == 0) {
449  CHECK(!dynamic_cast<const Analyzer::AggExpr*>(target_expr));
450  ++slot_index_counter;
451  ++target_index_counter;
452  return;
453  }
454  if (dynamic_cast<const Analyzer::UOper*>(target_expr) &&
455  static_cast<const Analyzer::UOper*>(target_expr)->get_optype() == kUNNEST) {
456  throw std::runtime_error("UNNEST not supported in the projection list yet.");
457  }
458  if ((executor->plan_state_->isLazyFetchColumn(target_expr) || !is_group_by) &&
459  (static_cast<size_t>(query_mem_desc.getPaddedSlotWidthBytes(slot_index_counter)) <
460  sizeof(int64_t)) &&
461  !is_columnar_projection(query_mem_desc)) {
462  // TODO(miyu): enable different byte width in the layout w/o padding
464  }
465 
466  auto target_info = get_target_info(target_expr, g_bigint_count);
467  auto arg_expr = agg_arg(target_expr);
468  if (arg_expr) {
470  target_info.skip_null_val = false;
471  } else if (query_mem_desc.getQueryDescriptionType() ==
473  !arg_expr->get_type_info().is_varlen()) {
474  // TODO: COUNT is currently not null-aware for varlen types. Need to add proper code
475  // generation for handling varlen nulls.
476  target_info.skip_null_val = true;
477  } else if (constrained_not_null(arg_expr, ra_exe_unit.quals)) {
478  target_info.skip_null_val = false;
479  }
480  }
481 
482  if (!(query_mem_desc.getQueryDescriptionType() ==
486  sample_exprs_to_codegen.emplace_back(target_expr,
487  target_info,
488  slot_index_counter,
489  target_index_counter++,
490  is_group_by);
491  } else {
492  target_exprs_to_codegen.emplace_back(target_expr,
493  target_info,
494  slot_index_counter,
495  target_index_counter++,
496  is_group_by);
497  }
498 
499  const auto agg_fn_names = agg_fn_base_names(target_info);
500  slot_index_counter += agg_fn_names.size();
501 }
502 
503 namespace {
504 
506  const QueryMemoryDescriptor& query_mem_desc) {
507  const bool is_group_by{query_mem_desc.isGroupBy()};
508  if (target_info.agg_kind == kSAMPLE && target_info.sql_type.is_string() &&
509  target_info.sql_type.get_compression() != kENCODING_NONE) {
510  return get_agg_initial_val(target_info.agg_kind,
511  target_info.sql_type,
512  is_group_by,
513  query_mem_desc.getCompactByteWidth());
514  }
515  return 0;
516 }
517 
518 } // namespace
519 
521  GroupByAndAggregate* group_by_and_agg,
522  Executor* executor,
523  const QueryMemoryDescriptor& query_mem_desc,
524  const CompilationOptions& co,
525  const std::tuple<llvm::Value*, llvm::Value*>& agg_out_ptr_w_idx,
526  const std::vector<llvm::Value*>& agg_out_vec,
527  llvm::Value* output_buffer_byte_stream,
528  llvm::Value* out_row_idx,
529  GroupByAndAggregate::DiamondCodegen& diamond_codegen) const {
530  CHECK(group_by_and_agg);
531  CHECK(executor);
532 
533  for (const auto& target_expr_codegen : target_exprs_to_codegen) {
534  target_expr_codegen.codegen(group_by_and_agg,
535  executor,
536  query_mem_desc,
537  co,
538  agg_out_ptr_w_idx,
539  agg_out_vec,
540  output_buffer_byte_stream,
541  out_row_idx,
542  diamond_codegen);
543  }
544  if (!sample_exprs_to_codegen.empty()) {
545  codegenSampleExpressions(group_by_and_agg,
546  executor,
547  query_mem_desc,
548  co,
549  agg_out_ptr_w_idx,
550  agg_out_vec,
551  output_buffer_byte_stream,
552  out_row_idx,
553  diamond_codegen);
554  }
555 }
556 
558  GroupByAndAggregate* group_by_and_agg,
559  Executor* executor,
560  const QueryMemoryDescriptor& query_mem_desc,
561  const CompilationOptions& co,
562  const std::tuple<llvm::Value*, llvm::Value*>& agg_out_ptr_w_idx,
563  const std::vector<llvm::Value*>& agg_out_vec,
564  llvm::Value* output_buffer_byte_stream,
565  llvm::Value* out_row_idx,
566  GroupByAndAggregate::DiamondCodegen& diamond_codegen) const {
567  CHECK(!sample_exprs_to_codegen.empty());
569  if (sample_exprs_to_codegen.size() == 1 &&
570  !sample_exprs_to_codegen.front().target_info.sql_type.is_varlen()) {
571  codegenSingleSlotSampleExpression(group_by_and_agg,
572  executor,
573  query_mem_desc,
574  co,
575  agg_out_ptr_w_idx,
576  agg_out_vec,
577  output_buffer_byte_stream,
578  out_row_idx,
579  diamond_codegen);
580  } else {
581  codegenMultiSlotSampleExpressions(group_by_and_agg,
582  executor,
583  query_mem_desc,
584  co,
585  agg_out_ptr_w_idx,
586  agg_out_vec,
587  output_buffer_byte_stream,
588  out_row_idx,
589  diamond_codegen);
590  }
591 }
592 
594  GroupByAndAggregate* group_by_and_agg,
595  Executor* executor,
596  const QueryMemoryDescriptor& query_mem_desc,
597  const CompilationOptions& co,
598  const std::tuple<llvm::Value*, llvm::Value*>& agg_out_ptr_w_idx,
599  const std::vector<llvm::Value*>& agg_out_vec,
600  llvm::Value* output_buffer_byte_stream,
601  llvm::Value* out_row_idx,
602  GroupByAndAggregate::DiamondCodegen& diamond_codegen) const {
603  CHECK_EQ(size_t(1), sample_exprs_to_codegen.size());
604  CHECK(!sample_exprs_to_codegen.front().target_info.sql_type.is_varlen());
606  // no need for the atomic if we only have one SAMPLE target
607  sample_exprs_to_codegen.front().codegen(group_by_and_agg,
608  executor,
609  query_mem_desc,
610  co,
611  agg_out_ptr_w_idx,
612  agg_out_vec,
613  output_buffer_byte_stream,
614  out_row_idx,
615  diamond_codegen);
616 }
617 
619  GroupByAndAggregate* group_by_and_agg,
620  Executor* executor,
621  const QueryMemoryDescriptor& query_mem_desc,
622  const CompilationOptions& co,
623  const std::tuple<llvm::Value*, llvm::Value*>& agg_out_ptr_w_idx,
624  const std::vector<llvm::Value*>& agg_out_vec,
625  llvm::Value* output_buffer_byte_stream,
626  llvm::Value* out_row_idx,
627  GroupByAndAggregate::DiamondCodegen& diamond_codegen) const {
628  CHECK(sample_exprs_to_codegen.size() > 1 ||
629  sample_exprs_to_codegen.front().target_info.sql_type.is_varlen());
631  const auto& first_sample_expr = sample_exprs_to_codegen.front();
632  auto target_lvs = group_by_and_agg->codegenAggArg(first_sample_expr.target_expr, co);
633  CHECK_GE(target_lvs.size(), size_t(1));
634 
635  const auto init_val =
636  get_initial_agg_val(first_sample_expr.target_info, query_mem_desc);
637 
638  llvm::Value* agg_col_ptr{nullptr};
639  if (is_group_by) {
640  const auto agg_column_size_bytes =
641  query_mem_desc.isLogicalSizedColumnsAllowed() &&
642  !first_sample_expr.target_info.sql_type.is_varlen()
643  ? first_sample_expr.target_info.sql_type.get_size()
644  : sizeof(int64_t);
645  agg_col_ptr = group_by_and_agg->codegenAggColumnPtr(output_buffer_byte_stream,
646  out_row_idx,
647  agg_out_ptr_w_idx,
648  query_mem_desc,
649  agg_column_size_bytes,
650  first_sample_expr.base_slot_index,
651  first_sample_expr.target_idx);
652  } else {
653  CHECK_LT(static_cast<size_t>(first_sample_expr.base_slot_index), agg_out_vec.size());
654  agg_col_ptr =
655  executor->castToIntPtrTyIn(agg_out_vec[first_sample_expr.base_slot_index], 64);
656  }
657 
658  auto sample_cas_lv = codegenSlotEmptyKey(agg_col_ptr, target_lvs, executor, init_val);
659 
661  sample_cas_lv, executor, false, "sample_valcheck", &diamond_codegen, false);
662 
663  for (const auto& target_expr_codegen : sample_exprs_to_codegen) {
664  target_expr_codegen.codegen(group_by_and_agg,
665  executor,
666  query_mem_desc,
667  co,
668  agg_out_ptr_w_idx,
669  agg_out_vec,
670  output_buffer_byte_stream,
671  out_row_idx,
672  diamond_codegen,
673  &sample_cfg);
674  }
675 }
676 
678  llvm::Value* agg_col_ptr,
679  std::vector<llvm::Value*>& target_lvs,
680  Executor* executor,
681  const int64_t init_val) const {
682  const auto& first_sample_expr = sample_exprs_to_codegen.front();
683  const auto first_sample_slot_bytes =
684  first_sample_expr.target_info.sql_type.is_varlen()
685  ? sizeof(int64_t)
686  : first_sample_expr.target_info.sql_type.get_size();
687  llvm::Value* target_lv_casted{nullptr};
688  // deciding whether proper casting is required for the first sample's slot:
689  if (first_sample_expr.target_info.sql_type.is_varlen()) {
690  target_lv_casted =
691  LL_BUILDER.CreatePtrToInt(target_lvs.front(), llvm::Type::getInt64Ty(LL_CONTEXT));
692  } else if (first_sample_expr.target_info.sql_type.is_fp()) {
693  // Initialization value for SAMPLE on a float column should be 0
694  CHECK_EQ(init_val, 0);
695  if (query_mem_desc.isLogicalSizedColumnsAllowed()) {
696  target_lv_casted = executor->cgen_state_->ir_builder_.CreateFPToSI(
697  target_lvs.front(),
698  first_sample_slot_bytes == sizeof(float) ? llvm::Type::getInt32Ty(LL_CONTEXT)
699  : llvm::Type::getInt64Ty(LL_CONTEXT));
700  } else {
701  target_lv_casted = executor->cgen_state_->ir_builder_.CreateFPToSI(
702  target_lvs.front(), llvm::Type::getInt64Ty(LL_CONTEXT));
703  }
704  } else if (first_sample_slot_bytes != sizeof(int64_t) &&
705  !query_mem_desc.isLogicalSizedColumnsAllowed()) {
706  target_lv_casted =
707  executor->cgen_state_->ir_builder_.CreateCast(llvm::Instruction::CastOps::SExt,
708  target_lvs.front(),
709  llvm::Type::getInt64Ty(LL_CONTEXT));
710  } else {
711  target_lv_casted = target_lvs.front();
712  }
713 
714  std::string slot_empty_cas_func_name("slotEmptyKeyCAS");
715  llvm::Value* init_val_lv{LL_INT(init_val)};
716  if (query_mem_desc.isLogicalSizedColumnsAllowed() &&
717  !first_sample_expr.target_info.sql_type.is_varlen()) {
718  // add proper suffix to the function name:
719  switch (first_sample_slot_bytes) {
720  case 1:
721  slot_empty_cas_func_name += "_int8";
722  break;
723  case 2:
724  slot_empty_cas_func_name += "_int16";
725  break;
726  case 4:
727  slot_empty_cas_func_name += "_int32";
728  break;
729  case 8:
730  break;
731  default:
732  UNREACHABLE() << "Invalid slot size for slotEmptyKeyCAS function.";
733  break;
734  }
735  if (first_sample_slot_bytes != sizeof(int64_t)) {
736  init_val_lv = llvm::ConstantInt::get(
737  get_int_type(first_sample_slot_bytes * 8, LL_CONTEXT), init_val);
738  }
739  }
740 
741  auto sample_cas_lv = executor->cgen_state_->emitExternalCall(
742  slot_empty_cas_func_name,
743  llvm::Type::getInt1Ty(executor->cgen_state_->context_),
744  {agg_col_ptr, target_lv_casted, init_val_lv});
745  return sample_cas_lv;
746 }
#define LL_BUILDER
const Analyzer::Expr * agg_arg(const Analyzer::Expr *expr)
#define CHECK_EQ(x, y)
Definition: Logger.h:201
bool constrained_not_null(const Analyzer::Expr *expr, const std::list< std::shared_ptr< Analyzer::Expr >> &quals)
std::vector< std::string > agg_fn_base_names(const TargetInfo &target_info)
llvm::Value * codegenAggColumnPtr(llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const QueryMemoryDescriptor &query_mem_desc, const size_t chosen_bytes, const size_t agg_out_off, const size_t target_idx)
: returns the pointer to where the aggregation should be stored.
void codegenSampleExpressions(GroupByAndAggregate *group_by_and_agg, Executor *executor, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const std::vector< llvm::Value * > &agg_out_vec, llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, GroupByAndAggregate::DiamondCodegen &diamond_codegen) const
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
bool isLogicalSizedColumnsAllowed() const
SQLTypeInfo sql_type
Definition: TargetInfo.h:42
TargetInfo get_target_info(const PointerType target_expr, const bool bigint_count)
Definition: TargetInfo.h:66
llvm::Value * posArg(const Analyzer::Expr *) const
Definition: ColumnIR.cpp:503
#define UNREACHABLE()
Definition: Logger.h:237
#define CHECK_GE(x, y)
Definition: Logger.h:206
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
int64_t get_agg_initial_val(const SQLAgg agg, const SQLTypeInfo &ti, const bool enable_compaction, const unsigned min_byte_width_to_compact)
void checkErrorCode(llvm::Value *retCode)
bool needsUnnestDoublePatch(llvm::Value *val_ptr, const std::string &agg_base_name, const bool threads_share_memory, const CompilationOptions &co) const
bool takes_float_argument(const TargetInfo &target_info)
Definition: TargetInfo.h:121
bool skip_null_val
Definition: TargetInfo.h:44
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
std::string to_string(char const *&&v)
SQLTypeInfo agg_arg_type
Definition: TargetInfo.h:43
llvm::Value * codegenSlotEmptyKey(llvm::Value *agg_col_ptr, std::vector< llvm::Value * > &target_lvs, Executor *executor, const int64_t init_val) const
std::string patch_agg_fname(const std::string &agg_name)
Helpers for codegen of target expressions.
void codegenSingleSlotSampleExpression(GroupByAndAggregate *group_by_and_agg, Executor *executor, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const std::vector< llvm::Value * > &agg_out_vec, llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, GroupByAndAggregate::DiamondCodegen &diamond_codegen) const
size_t getColOnlyOffInBytes(const size_t col_idx) const
Definition: sqldefs.h:73
const SQLTypeInfo get_compact_type(const TargetInfo &target)
void codegen(GroupByAndAggregate *group_by_and_agg, Executor *executor, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const std::vector< llvm::Value * > &agg_out_vec, llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, GroupByAndAggregate::DiamondCodegen &diamond_codegen, GroupByAndAggregate::DiamondCodegen *sample_cfg=nullptr) const
bool is_agg
Definition: TargetInfo.h:40
CHECK(cgen_state)
void codegen(GroupByAndAggregate *group_by_and_agg, Executor *executor, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const std::vector< llvm::Value * > &agg_out_vec, llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, GroupByAndAggregate::DiamondCodegen &diamond_codegen) const
#define LL_INT(v)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
llvm::Value * convertNullIfAny(const SQLTypeInfo &arg_type, const TargetInfo &agg_info, llvm::Value *target)
int get_physical_coord_cols() const
Definition: sqltypes.h:362
bool g_bigint_count
Definition: sqldefs.h:75
#define LL_CONTEXT
bool is_distinct_target(const TargetInfo &target_info)
Definition: TargetInfo.h:117
void codegenCountDistinct(const size_t target_idx, const Analyzer::Expr *target_expr, std::vector< llvm::Value * > &agg_args, const QueryMemoryDescriptor &, const ExecutorDeviceType)
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
SQLAgg agg_kind
Definition: TargetInfo.h:41
bool is_geometry() const
Definition: sqltypes.h:489
ExecutorDeviceType device_type_
QueryDescriptionType getQueryDescriptionType() const
bool window_function_is_aggregate(const SqlWindowFunctionKind kind)
Definition: WindowContext.h:42
#define CHECK_LT(x, y)
Definition: Logger.h:203
static void resetWindowFunctionContext()
const Analyzer::Expr * target_expr
Definition: sqldefs.h:76
std::vector< llvm::Value * > codegenAggArg(const Analyzer::Expr *target_expr, const CompilationOptions &co)
llvm::Value * codegenWindowRowPointer(const Analyzer::WindowFunction *window_func, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, DiamondCodegen &diamond_codegen)
bool window_function_requires_peer_handling(const Analyzer::WindowFunction *window_func)
void codegenMultiSlotSampleExpressions(GroupByAndAggregate *group_by_and_agg, Executor *executor, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const std::vector< llvm::Value * > &agg_out_vec, llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, GroupByAndAggregate::DiamondCodegen &diamond_codegen) const
bool is_string() const
Definition: sqltypes.h:477
int64_t get_initial_agg_val(const TargetInfo &target_info, const QueryMemoryDescriptor &query_mem_desc)
GroupByMemSharing getGpuMemSharing() const
std::string numeric_type_name(const SQLTypeInfo &ti)
Definition: Execute.h:159
bool is_distinct
Definition: TargetInfo.h:45
void operator()(const Analyzer::Expr *target_expr, const Executor *executor, const CompilationOptions &co)
Definition: sqldefs.h:74
const int8_t getLogicalSlotWidthBytes(const size_t slot_idx) const
Definition: sqldefs.h:72
size_t getColOffInBytes(const size_t col_idx) const
bool is_columnar_projection(const QueryMemoryDescriptor &query_mem_desc)
static WindowFunctionContext * getActiveWindowFunctionContext()
bool is_agg_domain_range_equivalent(const SQLAgg &agg_kind)
Definition: TargetInfo.h:52