OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
IRCodegen.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "../Parser/ParserNode.h"
18 #include "CodeGenerator.h"
19 #include "Execute.h"
20 #include "ExternalExecutor.h"
21 #include "MaxwellCodegenPatch.h"
22 #include "RelAlgTranslator.h"
23 
25 
26 // Driver methods for the IR generation.
27 
29 
30 std::vector<llvm::Value*> CodeGenerator::codegen(const Analyzer::Expr* expr,
31  const bool fetch_columns,
32  const CompilationOptions& co) {
34  if (!expr) {
35  return {posArg(expr)};
36  }
37  auto bin_oper = dynamic_cast<const Analyzer::BinOper*>(expr);
38  if (bin_oper) {
39  return {codegen(bin_oper, co)};
40  }
41  auto u_oper = dynamic_cast<const Analyzer::UOper*>(expr);
42  if (u_oper) {
43  return {codegen(u_oper, co)};
44  }
45  auto geo_col_var = dynamic_cast<const Analyzer::GeoColumnVar*>(expr);
46  if (geo_col_var) {
47  // inherits from ColumnVar, so it is important we check this first
48  return codegenGeoColumnVar(geo_col_var, fetch_columns, co);
49  }
50  auto col_var = dynamic_cast<const Analyzer::ColumnVar*>(expr);
51  if (col_var) {
52  return codegenColumn(col_var, fetch_columns, co);
53  }
54  auto constant = dynamic_cast<const Analyzer::Constant*>(expr);
55  if (constant) {
56  const auto& ti = constant->get_type_info();
57  if (ti.get_type() == kNULLT) {
58  throw std::runtime_error(
59  "NULL type literals are not currently supported in this context.");
60  }
61  if (constant->get_is_null()) {
62  return {ti.is_fp()
63  ? static_cast<llvm::Value*>(executor_->cgen_state_->inlineFpNull(ti))
64  : static_cast<llvm::Value*>(executor_->cgen_state_->inlineIntNull(ti))};
65  }
66  if (ti.get_compression() == kENCODING_DICT) {
67  // The dictionary encoding case should be handled by the parent expression
68  // (cast, for now), here is too late to know the dictionary id if not already set
69  CHECK_NE(ti.getStringDictKey().dict_id, 0);
70  return {codegen(constant, ti.get_compression(), ti.getStringDictKey(), co)};
71  }
72  return {codegen(constant, ti.get_compression(), {}, co)};
73  }
74  auto case_expr = dynamic_cast<const Analyzer::CaseExpr*>(expr);
75  if (case_expr) {
76  return {codegen(case_expr, co)};
77  }
78  auto extract_expr = dynamic_cast<const Analyzer::ExtractExpr*>(expr);
79  if (extract_expr) {
80  return {codegen(extract_expr, co)};
81  }
82  auto dateadd_expr = dynamic_cast<const Analyzer::DateaddExpr*>(expr);
83  if (dateadd_expr) {
84  return {codegen(dateadd_expr, co)};
85  }
86  auto datediff_expr = dynamic_cast<const Analyzer::DatediffExpr*>(expr);
87  if (datediff_expr) {
88  return {codegen(datediff_expr, co)};
89  }
90  auto datetrunc_expr = dynamic_cast<const Analyzer::DatetruncExpr*>(expr);
91  if (datetrunc_expr) {
92  return {codegen(datetrunc_expr, co)};
93  }
94  auto charlength_expr = dynamic_cast<const Analyzer::CharLengthExpr*>(expr);
95  if (charlength_expr) {
96  return {codegen(charlength_expr, co)};
97  }
98  auto keyforstring_expr = dynamic_cast<const Analyzer::KeyForStringExpr*>(expr);
99  if (keyforstring_expr) {
100  return {codegen(keyforstring_expr, co)};
101  }
102  auto sample_ratio_expr = dynamic_cast<const Analyzer::SampleRatioExpr*>(expr);
103  if (sample_ratio_expr) {
104  return {codegen(sample_ratio_expr, co)};
105  }
106  auto string_oper_expr = dynamic_cast<const Analyzer::StringOper*>(expr);
107  if (string_oper_expr) {
108  return {codegen(string_oper_expr, co)};
109  }
110  auto cardinality_expr = dynamic_cast<const Analyzer::CardinalityExpr*>(expr);
111  if (cardinality_expr) {
112  return {codegen(cardinality_expr, co)};
113  }
114  auto like_expr = dynamic_cast<const Analyzer::LikeExpr*>(expr);
115  if (like_expr) {
116  return {codegen(like_expr, co)};
117  }
118  auto regexp_expr = dynamic_cast<const Analyzer::RegexpExpr*>(expr);
119  if (regexp_expr) {
120  return {codegen(regexp_expr, co)};
121  }
122  auto width_bucket_expr = dynamic_cast<const Analyzer::WidthBucketExpr*>(expr);
123  if (width_bucket_expr) {
124  return {codegen(width_bucket_expr, co)};
125  }
126  auto likelihood_expr = dynamic_cast<const Analyzer::LikelihoodExpr*>(expr);
127  if (likelihood_expr) {
128  return {codegen(likelihood_expr->get_arg(), fetch_columns, co)};
129  }
130  auto in_expr = dynamic_cast<const Analyzer::InValues*>(expr);
131  if (in_expr) {
132  return {codegen(in_expr, co)};
133  }
134  auto in_integer_set_expr = dynamic_cast<const Analyzer::InIntegerSet*>(expr);
135  if (in_integer_set_expr) {
136  return {codegen(in_integer_set_expr, co)};
137  }
138  auto function_oper_with_custom_type_handling_expr =
139  dynamic_cast<const Analyzer::FunctionOperWithCustomTypeHandling*>(expr);
140  if (function_oper_with_custom_type_handling_expr) {
142  function_oper_with_custom_type_handling_expr, co)};
143  }
144  auto array_oper_expr = dynamic_cast<const Analyzer::ArrayExpr*>(expr);
145  if (array_oper_expr) {
146  return {codegenArrayExpr(array_oper_expr, co)};
147  }
148  auto geo_uop = dynamic_cast<const Analyzer::GeoUOper*>(expr);
149  if (geo_uop) {
150  return {codegenGeoUOper(geo_uop, co)};
151  }
152  auto geo_binop = dynamic_cast<const Analyzer::GeoBinOper*>(expr);
153  if (geo_binop) {
154  return {codegenGeoBinOper(geo_binop, co)};
155  }
156  auto function_oper_expr = dynamic_cast<const Analyzer::FunctionOper*>(expr);
157  if (function_oper_expr) {
158  return {codegenFunctionOper(function_oper_expr, co)};
159  }
160  auto geo_expr = dynamic_cast<const Analyzer::GeoExpr*>(expr);
161  if (geo_expr) {
162  return codegenGeoExpr(geo_expr, co);
163  }
164  if (dynamic_cast<const Analyzer::OffsetInFragment*>(expr)) {
165  return {posArg(nullptr)};
166  }
167  if (dynamic_cast<const Analyzer::WindowFunction*>(expr)) {
168  throw NativeExecutionError("Window expression not supported in this context");
169  }
170  abort();
171 }
172 
173 llvm::Value* CodeGenerator::codegen(const Analyzer::BinOper* bin_oper,
174  const CompilationOptions& co) {
176  const auto optype = bin_oper->get_optype();
177  if (IS_ARITHMETIC(optype)) {
178  return codegenArith(bin_oper, co);
179  }
180  if (IS_COMPARISON(optype)) {
181  return codegenCmp(bin_oper, co);
182  }
183  if (IS_LOGIC(optype)) {
184  return codegenLogical(bin_oper, co);
185  }
186  if (optype == kARRAY_AT) {
187  return codegenArrayAt(bin_oper, co);
188  }
189  abort();
190 }
191 
192 llvm::Value* CodeGenerator::codegen(const Analyzer::UOper* u_oper,
193  const CompilationOptions& co) {
195  const auto optype = u_oper->get_optype();
196  switch (optype) {
197  case kNOT: {
198  return codegenLogical(u_oper, co);
199  }
200  case kCAST: {
201  return codegenCast(u_oper, co);
202  }
203  case kUMINUS: {
204  return codegenUMinus(u_oper, co);
205  }
206  case kISNULL: {
207  return codegenIsNull(u_oper, co);
208  }
209  case kUNNEST:
210  return codegenUnnest(u_oper, co);
211  default:
212  UNREACHABLE();
213  }
214  return nullptr;
215 }
216 
218  const CompilationOptions& co) {
220  auto input_expr = expr->get_arg();
221  CHECK(input_expr);
222 
223  auto double_lv = codegen(input_expr, true, co);
224  CHECK_EQ(size_t(1), double_lv.size());
225 
226  std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
227  const bool is_nullable = !input_expr->get_type_info().get_notnull();
228  if (is_nullable) {
229  nullcheck_codegen = std::make_unique<NullCheckCodegen>(cgen_state_,
230  executor(),
231  double_lv.front(),
232  input_expr->get_type_info(),
233  "sample_ratio_nullcheck");
234  }
235  CHECK_EQ(input_expr->get_type_info().get_type(), kDOUBLE);
236  std::vector<llvm::Value*> args{double_lv[0], posArg(nullptr)};
237  auto ret = cgen_state_->emitCall("sample_ratio", args);
238  if (nullcheck_codegen) {
239  ret = nullcheck_codegen->finalize(ll_bool(false, cgen_state_->context_), ret);
240  }
241  return ret;
242 }
243 
245  const CompilationOptions& co) {
247  auto target_value_expr = expr->get_target_value();
248  auto lower_bound_expr = expr->get_lower_bound();
249  auto upper_bound_expr = expr->get_upper_bound();
250  auto partition_count_expr = expr->get_partition_count();
251  CHECK(target_value_expr);
252  CHECK(lower_bound_expr);
253  CHECK(upper_bound_expr);
254  CHECK(partition_count_expr);
255 
256  auto is_constant_expr = [](const Analyzer::Expr* expr) {
257  auto target_expr = expr;
258  if (auto cast_expr = dynamic_cast<const Analyzer::UOper*>(expr)) {
259  if (cast_expr->get_optype() == SQLOps::kCAST) {
260  target_expr = cast_expr->get_operand();
261  }
262  }
263  // there are more complex constant expr like 1+2, 1/2*3, and so on
264  // but when considering a typical usage of width_bucket function
265  // it is sufficient to consider a singleton constant expr
266  auto constant_expr = dynamic_cast<const Analyzer::Constant*>(target_expr);
267  if (constant_expr) {
268  return true;
269  }
270  return false;
271  };
272  if (is_constant_expr(lower_bound_expr) && is_constant_expr(upper_bound_expr) &&
273  is_constant_expr(partition_count_expr)) {
274  expr->set_constant_expr();
275  }
276  // compute width_bucket's expresn range and check the possibility of avoiding oob check
277  auto col_range =
278  getExpressionRange(expr,
280  executor_,
281  boost::make_optional(plan_state_->getSimpleQuals()));
282  // check whether target_expr is valid
283  if (col_range.getType() == ExpressionRangeType::Integer &&
284  !expr->can_skip_out_of_bound_check() && col_range.getIntMin() > 0 &&
285  col_range.getIntMax() <= expr->get_partition_count_val()) {
286  // check whether target_col is not-nullable or has filter expr on it
287  if (!col_range.hasNulls()) {
288  // Even if the target_expr has its filter expression, target_col_range may exactly
289  // the same with the col_range of the target_expr's operand col,
290  // i.e., SELECT WIDTH_BUCKET(v1, 1, 10, 10) FROM T WHERE v1 != 1;
291  // In that query, col_range of v1 with/without considering the filter expression
292  // v1 != 1 have exactly the same col ranges, so we cannot recognize the existence
293  // of the filter expression based on them. Also, is (not) null is located in
294  // FilterNode, so we cannot trace it in here.
295  // todo (yoonmin): relax this to allow skipping oob check more cases
296  expr->skip_out_of_bound_check();
297  }
298  }
299  return expr->is_constant_expr() ? codegenConstantWidthBucketExpr(expr, co)
300  : codegenWidthBucketExpr(expr, co);
301 }
302 
304  const Analyzer::WidthBucketExpr* expr,
305  const CompilationOptions& co) {
306  auto target_value_expr = expr->get_target_value();
307  auto lower_bound_expr = expr->get_lower_bound();
308  auto upper_bound_expr = expr->get_upper_bound();
309  auto partition_count_expr = expr->get_partition_count();
310 
311  auto num_partitions = expr->get_partition_count_val();
312  if (num_partitions < 1 || num_partitions > INT32_MAX) {
313  throw std::runtime_error(
314  "PARTITION_COUNT expression of width_bucket function should be in a valid "
315  "range: 0 < PARTITION_COUNT <= 2147483647");
316  }
317  double lower = expr->get_bound_val(lower_bound_expr);
318  double upper = expr->get_bound_val(upper_bound_expr);
319  if (lower == upper) {
320  throw std::runtime_error(
321  "LOWER_BOUND and UPPER_BOUND expressions of width_bucket function cannot have "
322  "the same constant value");
323  }
324  if (lower == NULL_DOUBLE || upper == NULL_DOUBLE) {
325  throw std::runtime_error(
326  "Both LOWER_BOUND and UPPER_BOUND of width_bucket function should be finite "
327  "numeric constants.");
328  }
329 
330  bool const reversed = lower > upper;
331  double const scale_factor = num_partitions / (reversed ? lower - upper : upper - lower);
332  std::string func_name = reversed ? "width_bucket_reversed" : "width_bucket";
333 
334  auto get_double_constant_lvs = [this, &co](double const_val) {
335  Datum d;
336  d.doubleval = const_val;
337  auto double_const_expr =
338  makeExpr<Analyzer::Constant>(SQLTypeInfo(kDOUBLE, false), false, d);
339  return codegen(double_const_expr.get(), false, co);
340  };
341 
342  auto target_value_ti = target_value_expr->get_type_info();
343  auto target_value_expr_lvs = codegen(target_value_expr, true, co);
344  CHECK_EQ(size_t(1), target_value_expr_lvs.size());
345  auto lower_expr_lvs = codegen(lower_bound_expr, true, co);
346  CHECK_EQ(size_t(1), lower_expr_lvs.size());
347  auto scale_factor_lvs = get_double_constant_lvs(scale_factor);
348  CHECK_EQ(size_t(1), scale_factor_lvs.size());
349 
350  std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
351  lower_expr_lvs[0]};
352  if (expr->can_skip_out_of_bound_check()) {
353  func_name += "_no_oob_check";
354  width_bucket_args.push_back(scale_factor_lvs[0]);
355  } else {
356  auto upper_expr_lvs = codegen(upper_bound_expr, true, co);
357  CHECK_EQ(size_t(1), upper_expr_lvs.size());
358  auto partition_count_expr_lvs = codegen(partition_count_expr, true, co);
359  CHECK_EQ(size_t(1), partition_count_expr_lvs.size());
360  width_bucket_args.push_back(upper_expr_lvs[0]);
361  width_bucket_args.push_back(scale_factor_lvs[0]);
362  width_bucket_args.push_back(partition_count_expr_lvs[0]);
363  if (!target_value_ti.get_notnull()) {
364  func_name += "_nullable";
365  auto translated_null_value = target_value_ti.is_fp()
366  ? inline_fp_null_val(target_value_ti)
367  : inline_int_null_val(target_value_ti);
368  auto null_value_lvs = get_double_constant_lvs(translated_null_value);
369  CHECK_EQ(size_t(1), null_value_lvs.size());
370  width_bucket_args.push_back(null_value_lvs[0]);
371  }
372  }
373  return cgen_state_->emitCall(func_name, width_bucket_args);
374 }
375 
377  const CompilationOptions& co) {
378  auto target_value_expr = expr->get_target_value();
379  auto lower_bound_expr = expr->get_lower_bound();
380  auto upper_bound_expr = expr->get_upper_bound();
381  auto partition_count_expr = expr->get_partition_count();
382 
383  std::string func_name = "width_bucket_expr";
384  bool nullable_expr = false;
385  if (expr->can_skip_out_of_bound_check()) {
386  func_name += "_no_oob_check";
387  } else if (!target_value_expr->get_type_info().get_notnull()) {
388  func_name += "_nullable";
389  nullable_expr = true;
390  }
391 
392  auto target_value_expr_lvs = codegen(target_value_expr, true, co);
393  CHECK_EQ(size_t(1), target_value_expr_lvs.size());
394  auto lower_bound_expr_lvs = codegen(lower_bound_expr, true, co);
395  CHECK_EQ(size_t(1), lower_bound_expr_lvs.size());
396  auto upper_bound_expr_lvs = codegen(upper_bound_expr, true, co);
397  CHECK_EQ(size_t(1), upper_bound_expr_lvs.size());
398  auto partition_count_expr_lvs = codegen(partition_count_expr, true, co);
399  CHECK_EQ(size_t(1), partition_count_expr_lvs.size());
400  auto target_value_ti = target_value_expr->get_type_info();
401  auto null_value_lv = cgen_state_->inlineFpNull(target_value_ti);
402 
403  // check partition count : 1 ~ INT32_MAX
404  // INT32_MAX will be checked during casting by OVERFLOW checking step
405  auto partition_count_ti = partition_count_expr->get_type_info();
406  CHECK(partition_count_ti.is_integer());
407  auto int32_ti = SQLTypeInfo(kINT, partition_count_ti.get_notnull());
408  auto partition_count_expr_lv =
409  codegenCastBetweenIntTypes(partition_count_expr_lvs[0],
410  partition_count_ti,
411  int32_ti,
412  partition_count_ti.get_size() < int32_ti.get_size());
413  llvm::Value* chosen_min = cgen_state_->llInt(static_cast<int32_t>(0));
414  llvm::Value* partition_count_min =
415  cgen_state_->ir_builder_.CreateICmpSLE(partition_count_expr_lv, chosen_min);
416  llvm::BasicBlock* width_bucket_partition_count_ok_bb =
417  llvm::BasicBlock::Create(cgen_state_->context_,
418  "width_bucket_partition_count_ok_bb",
420  llvm::BasicBlock* width_bucket_argument_check_fail_bb =
421  llvm::BasicBlock::Create(cgen_state_->context_,
422  "width_bucket_argument_check_fail_bb",
424  cgen_state_->ir_builder_.CreateCondBr(partition_count_min,
425  width_bucket_argument_check_fail_bb,
426  width_bucket_partition_count_ok_bb);
427  cgen_state_->ir_builder_.SetInsertPoint(width_bucket_argument_check_fail_bb);
428  cgen_state_->ir_builder_.CreateRet(
430  cgen_state_->ir_builder_.SetInsertPoint(width_bucket_partition_count_ok_bb);
431 
432  llvm::BasicBlock* width_bucket_bound_check_ok_bb =
433  llvm::BasicBlock::Create(cgen_state_->context_,
434  "width_bucket_bound_check_ok_bb",
436  llvm::Value* bound_check{nullptr};
437  if (lower_bound_expr->get_type_info().get_notnull() &&
438  upper_bound_expr->get_type_info().get_notnull()) {
439  bound_check = cgen_state_->ir_builder_.CreateFCmpOEQ(
440  lower_bound_expr_lvs[0], upper_bound_expr_lvs[0], "bound_check");
441  } else {
442  std::vector<llvm::Value*> bound_check_args{
443  lower_bound_expr_lvs[0],
444  upper_bound_expr_lvs[0],
445  null_value_lv,
446  cgen_state_->llInt(static_cast<int8_t>(1))};
447  bound_check = toBool(cgen_state_->emitCall("eq_double_nullable", bound_check_args));
448  }
449  cgen_state_->ir_builder_.CreateCondBr(
450  bound_check, width_bucket_argument_check_fail_bb, width_bucket_bound_check_ok_bb);
451  cgen_state_->ir_builder_.SetInsertPoint(width_bucket_bound_check_ok_bb);
453  auto reversed_expr = toBool(codegenCmp(SQLOps::kGT,
454  kONE,
455  lower_bound_expr_lvs,
456  lower_bound_expr->get_type_info(),
457  upper_bound_expr,
458  co));
459  auto lower_bound_expr_lv = lower_bound_expr_lvs[0];
460  auto upper_bound_expr_lv = upper_bound_expr_lvs[0];
461  std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
462  reversed_expr,
463  lower_bound_expr_lv,
464  upper_bound_expr_lv,
465  partition_count_expr_lv};
466  if (nullable_expr) {
467  width_bucket_args.push_back(null_value_lv);
468  }
469  return cgen_state_->emitCall(func_name, width_bucket_args);
470 }
471 
472 namespace {
473 
475  const std::shared_ptr<Analyzer::Expr>& qual) {
476  const auto qual_cf = qual_to_conjunctive_form(qual);
477  ra_exe_unit.simple_quals.insert(ra_exe_unit.simple_quals.end(),
478  qual_cf.simple_quals.begin(),
479  qual_cf.simple_quals.end());
480  ra_exe_unit.quals.insert(
481  ra_exe_unit.quals.end(), qual_cf.quals.begin(), qual_cf.quals.end());
482 }
483 
485  const ExecutionOptions& eo,
486  const std::vector<InputTableInfo>& query_infos,
487  const size_t level_idx,
488  const std::string& fail_reason) {
489  if (ra_exe_unit.input_descs.size() < 2) {
490  // We only support loop join at the end of folded joins
491  // where ra_exe_unit.input_descs.size() > 2 for now.
492  throw std::runtime_error("Hash join failed, reason(s): " + fail_reason +
493  " | Incorrect # tables for executing loop join");
494  }
495  const auto loop_join_size = get_loop_join_size(query_infos, ra_exe_unit);
496  const bool has_loop_size_hint =
498  const size_t loop_join_size_threshold =
499  has_loop_size_hint ? ra_exe_unit.query_hint.loop_join_inner_table_max_num_rows
501  if (eo.allow_loop_joins) {
502  if (has_loop_size_hint && loop_join_size_threshold < loop_join_size) {
503  throw std::runtime_error(
504  "Hash join failed, reason(s): " + fail_reason +
505  " | Cannot fall back to loop join for non-trivial inner table size");
506  }
507  return;
508  }
509  if (level_idx + 1 != ra_exe_unit.join_quals.size()) {
510  throw std::runtime_error(
511  "Hash join failed, reason(s): " + fail_reason +
512  " | Cannot fall back to loop join for intermediate join quals");
513  }
514  if (loop_join_size_threshold < loop_join_size) {
515  throw std::runtime_error(
516  "Hash join failed, reason(s): " + fail_reason +
517  " | Cannot fall back to loop join for non-trivial inner table size");
518  }
519  if (ra_exe_unit.query_hint.isHintRegistered(kDisableLoopJoin)) {
520  throw std::runtime_error("Hash join failed, reason(s): " + fail_reason +
521  " | Loop join is disabled by user");
522  }
523 }
524 
525 void check_valid_join_qual(std::shared_ptr<Analyzer::BinOper>& bin_oper) {
526  // check whether a join qual is valid before entering the hashtable build and codegen
527 
528  auto lhs_cv = dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_left_operand());
529  auto rhs_cv = dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_right_operand());
530  if (lhs_cv && rhs_cv && !bin_oper->is_overlaps_oper()) {
531  auto lhs_type = lhs_cv->get_type_info().get_type();
532  auto rhs_type = rhs_cv->get_type_info().get_type();
533  // check #1. avoid a join btw full array columns
534  if (lhs_type == SQLTypes::kARRAY && rhs_type == SQLTypes::kARRAY) {
535  throw std::runtime_error(
536  "Join operation between full array columns (i.e., R.arr = S.arr) instead of "
537  "indexed array columns (i.e., R.arr[1] = S.arr[2]) is not supported yet.");
538  }
539  }
540 }
541 
542 } // namespace
543 
544 std::vector<JoinLoop> Executor::buildJoinLoops(
545  RelAlgExecutionUnit& ra_exe_unit,
546  const CompilationOptions& co,
547  const ExecutionOptions& eo,
548  const std::vector<InputTableInfo>& query_infos,
549  ColumnCacheMap& column_cache) {
550  INJECT_TIMER(buildJoinLoops);
552  std::vector<JoinLoop> join_loops;
553  for (size_t level_idx = 0, current_hash_table_idx = 0;
554  level_idx < ra_exe_unit.join_quals.size();
555  ++level_idx) {
556  const auto& current_level_join_conditions = ra_exe_unit.join_quals[level_idx];
557  std::vector<std::string> fail_reasons;
558  const auto current_level_hash_table =
559  buildCurrentLevelHashTable(current_level_join_conditions,
560  level_idx,
561  ra_exe_unit,
562  co,
563  query_infos,
564  column_cache,
565  fail_reasons);
566  const auto found_outer_join_matches_cb =
567  [this, level_idx](llvm::Value* found_outer_join_matches) {
571  found_outer_join_matches;
572  };
573  const auto is_deleted_cb = buildIsDeletedCb(ra_exe_unit, level_idx, co);
574  auto rem_left_join_quals_it =
576  bool has_remaining_left_join_quals =
577  rem_left_join_quals_it != plan_state_->left_join_non_hashtable_quals_.end() &&
578  !rem_left_join_quals_it->second.empty();
579  const auto outer_join_condition_remaining_quals_cb =
580  [this, level_idx, &co](const std::vector<llvm::Value*>& prev_iters) {
581  // when we have multiple quals for the left join in the current join level
582  // we first try to build a hashtable by using one of the possible qual,
583  // and deal with remaining quals as extra join conditions
584  FetchCacheAnchor anchor(cgen_state_.get());
585  addJoinLoopIterator(prev_iters, level_idx + 1);
586  llvm::Value* left_join_cond = cgen_state_->llBool(true);
587  CodeGenerator code_generator(this);
588  auto it = plan_state_->left_join_non_hashtable_quals_.find(level_idx);
589  if (it != plan_state_->left_join_non_hashtable_quals_.end()) {
590  for (auto expr : it->second) {
591  left_join_cond = cgen_state_->ir_builder_.CreateAnd(
592  left_join_cond,
593  code_generator.toBool(
594  code_generator.codegen(expr.get(), true, co).front()));
595  }
596  }
597  return left_join_cond;
598  };
599  if (current_level_hash_table) {
600  const auto hoisted_filters_cb = buildHoistLeftHandSideFiltersCb(
601  ra_exe_unit, level_idx, current_level_hash_table->getInnerTableId(), co);
602  if (current_level_hash_table->getHashType() == HashType::OneToOne) {
603  join_loops.emplace_back(
604  /*kind=*/JoinLoopKind::Singleton,
605  /*type=*/current_level_join_conditions.type,
606  /*iteration_domain_codegen=*/
607  [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
608  const std::vector<llvm::Value*>& prev_iters) {
609  addJoinLoopIterator(prev_iters, level_idx);
610  JoinLoopDomain domain{{0}};
611  domain.slot_lookup_result =
612  current_level_hash_table->codegenSlot(co, current_hash_table_idx);
613  return domain;
614  },
615  /*outer_condition_match=*/
616  current_level_join_conditions.type == JoinType::LEFT &&
617  has_remaining_left_join_quals
618  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
619  outer_join_condition_remaining_quals_cb)
620  : nullptr,
621  /*found_outer_matches=*/current_level_join_conditions.type == JoinType::LEFT
622  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
623  : nullptr,
624  /*hoisted_filters=*/hoisted_filters_cb,
625  /*is_deleted=*/is_deleted_cb,
626  /*nested_loop_join=*/false);
627  } else if (auto range_join_table =
628  dynamic_cast<RangeJoinHashTable*>(current_level_hash_table.get())) {
629  join_loops.emplace_back(
630  /* kind= */ JoinLoopKind::MultiSet,
631  /* type= */ current_level_join_conditions.type,
632  /* iteration_domain_codegen= */
633  [this,
634  range_join_table,
635  current_hash_table_idx,
636  level_idx,
637  current_level_hash_table,
638  &co](const std::vector<llvm::Value*>& prev_iters) {
639  addJoinLoopIterator(prev_iters, level_idx);
640  JoinLoopDomain domain{{0}};
641  CHECK(!prev_iters.empty());
642  const auto matching_set = range_join_table->codegenMatchingSetWithOffset(
643  co, current_hash_table_idx, prev_iters.back());
644  domain.values_buffer = matching_set.elements;
645  domain.element_count = matching_set.count;
646  return domain;
647  },
648  /* outer_condition_match= */
649  current_level_join_conditions.type == JoinType::LEFT
650  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
651  outer_join_condition_remaining_quals_cb)
652  : nullptr,
653  /* found_outer_matches= */
654  current_level_join_conditions.type == JoinType::LEFT
655  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
656  : nullptr,
657  /* hoisted_filters= */ nullptr, // <<! TODO
658  /* is_deleted= */ is_deleted_cb,
659  /*nested_loop_join=*/false);
660  } else {
661  join_loops.emplace_back(
662  /*kind=*/JoinLoopKind::Set,
663  /*type=*/current_level_join_conditions.type,
664  /*iteration_domain_codegen=*/
665  [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
666  const std::vector<llvm::Value*>& prev_iters) {
667  addJoinLoopIterator(prev_iters, level_idx);
668  JoinLoopDomain domain{{0}};
669  const auto matching_set = current_level_hash_table->codegenMatchingSet(
670  co, current_hash_table_idx);
671  domain.values_buffer = matching_set.elements;
672  domain.element_count = matching_set.count;
673  return domain;
674  },
675  /*outer_condition_match=*/
676  current_level_join_conditions.type == JoinType::LEFT
677  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
678  outer_join_condition_remaining_quals_cb)
679  : nullptr,
680  /*found_outer_matches=*/current_level_join_conditions.type == JoinType::LEFT
681  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
682  : nullptr,
683  /*hoisted_filters=*/hoisted_filters_cb,
684  /*is_deleted=*/is_deleted_cb,
685  /*nested_loop_join=*/false);
686  }
687  ++current_hash_table_idx;
688  } else {
689  const auto fail_reasons_str = current_level_join_conditions.quals.empty()
690  ? "No equijoin expression found"
691  : boost::algorithm::join(fail_reasons, " | ");
693  ra_exe_unit, eo, query_infos, level_idx, fail_reasons_str);
694  // Callback provided to the `JoinLoop` framework to evaluate the (outer) join
695  // condition.
696  VLOG(1) << "Unable to build hash table, falling back to loop join: "
697  << fail_reasons_str;
698  const auto outer_join_condition_cb =
699  [this, level_idx, &co, &current_level_join_conditions](
700  const std::vector<llvm::Value*>& prev_iters) {
701  // The values generated for the match path don't dominate all uses
702  // since on the non-match path nulls are generated. Reset the cache
703  // once the condition is generated to avoid incorrect reuse.
704  FetchCacheAnchor anchor(cgen_state_.get());
705  addJoinLoopIterator(prev_iters, level_idx + 1);
706  llvm::Value* left_join_cond = cgen_state_->llBool(true);
707  CodeGenerator code_generator(this);
708  for (auto expr : current_level_join_conditions.quals) {
709  left_join_cond = cgen_state_->ir_builder_.CreateAnd(
710  left_join_cond,
711  code_generator.toBool(
712  code_generator.codegen(expr.get(), true, co).front()));
713  }
714  return left_join_cond;
715  };
716  join_loops.emplace_back(
717  /*kind=*/JoinLoopKind::UpperBound,
718  /*type=*/current_level_join_conditions.type,
719  /*iteration_domain_codegen=*/
720  [this, level_idx](const std::vector<llvm::Value*>& prev_iters) {
721  addJoinLoopIterator(prev_iters, level_idx);
722  JoinLoopDomain domain{{0}};
723  auto* arg = get_arg_by_name(cgen_state_->row_func_, "num_rows_per_scan");
724  const auto rows_per_scan_ptr = cgen_state_->ir_builder_.CreateGEP(
725  arg->getType()->getScalarType()->getPointerElementType(),
726  arg,
727  cgen_state_->llInt(int32_t(level_idx + 1)));
728  domain.upper_bound = cgen_state_->ir_builder_.CreateLoad(
729  rows_per_scan_ptr->getType()->getPointerElementType(),
730  rows_per_scan_ptr,
731  "num_rows_per_scan");
732  return domain;
733  },
734  /*outer_condition_match=*/
735  current_level_join_conditions.type == JoinType::LEFT
736  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
737  outer_join_condition_cb)
738  : nullptr,
739  /*found_outer_matches=*/
740  current_level_join_conditions.type == JoinType::LEFT
741  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
742  : nullptr,
743  /*hoisted_filters=*/nullptr,
744  /*is_deleted=*/is_deleted_cb,
745  /*nested_loop_join=*/true);
746  }
747  }
748  return join_loops;
749 }
750 
751 namespace {
752 
753 class ExprTableIdVisitor : public ScalarExprVisitor<std::set<shared::TableKey>> {
754  protected:
755  std::set<shared::TableKey> visitColumnVar(
756  const Analyzer::ColumnVar* col_expr) const final {
757  return {col_expr->getTableKey()};
758  }
759 
760  std::set<shared::TableKey> visitFunctionOper(
761  const Analyzer::FunctionOper* func_expr) const final {
762  std::set<shared::TableKey> ret;
763  for (size_t i = 0; i < func_expr->getArity(); i++) {
764  ret = aggregateResult(ret, visit(func_expr->getArg(i)));
765  }
766  return ret;
767  }
768 
769  std::set<shared::TableKey> visitBinOper(const Analyzer::BinOper* bin_oper) const final {
770  std::set<shared::TableKey> ret;
771  ret = aggregateResult(ret, visit(bin_oper->get_left_operand()));
772  return aggregateResult(ret, visit(bin_oper->get_right_operand()));
773  }
774 
775  std::set<shared::TableKey> visitUOper(const Analyzer::UOper* u_oper) const final {
776  return visit(u_oper->get_operand());
777  }
778 
779  std::set<shared::TableKey> aggregateResult(
780  const std::set<shared::TableKey>& aggregate,
781  const std::set<shared::TableKey>& next_result) const final {
782  auto ret = aggregate; // copy
783  for (const auto& el : next_result) {
784  ret.insert(el);
785  }
786  return ret;
787  }
788 };
789 
790 } // namespace
791 
793  const RelAlgExecutionUnit& ra_exe_unit,
794  const size_t level_idx,
795  const shared::TableKey& inner_table_id,
796  const CompilationOptions& co) {
798  return nullptr;
799  }
800 
801  const auto& current_level_join_conditions = ra_exe_unit.join_quals[level_idx];
802  if (level_idx == 0 && current_level_join_conditions.type == JoinType::LEFT) {
803  const auto& condition = current_level_join_conditions.quals.front();
804  const auto bin_oper = dynamic_cast<const Analyzer::BinOper*>(condition.get());
805  CHECK(bin_oper) << condition->toString();
806  const auto rhs =
807  dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_right_operand());
808  const auto lhs =
809  dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_left_operand());
810  if (lhs && rhs && lhs->getTableKey() != rhs->getTableKey()) {
811  const Analyzer::ColumnVar* selected_lhs{nullptr};
812  // grab the left hand side column -- this is somewhat similar to normalize column
813  // pair, and a better solution may be to hoist that function out of the join
814  // framework and normalize columns at the top of build join loops
815  if (lhs->getTableKey() == inner_table_id) {
816  selected_lhs = rhs;
817  } else if (rhs->getTableKey() == inner_table_id) {
818  selected_lhs = lhs;
819  }
820  if (selected_lhs) {
821  std::list<std::shared_ptr<Analyzer::Expr>> hoisted_quals;
822  // get all LHS-only filters
823  auto should_hoist_qual = [&hoisted_quals](const auto& qual,
824  const shared::TableKey& table_key) {
825  CHECK(qual);
826 
827  ExprTableIdVisitor visitor;
828  const auto table_keys = visitor.visit(qual.get());
829  if (table_keys.size() == 1 && table_keys.find(table_key) != table_keys.end()) {
830  hoisted_quals.push_back(qual);
831  }
832  };
833  for (const auto& qual : ra_exe_unit.simple_quals) {
834  should_hoist_qual(qual, selected_lhs->getTableKey());
835  }
836  for (const auto& qual : ra_exe_unit.quals) {
837  should_hoist_qual(qual, selected_lhs->getTableKey());
838  }
839 
840  // build the filters callback and return it
841  if (!hoisted_quals.empty()) {
842  return [this, hoisted_quals, co](llvm::BasicBlock* true_bb,
843  llvm::BasicBlock* exit_bb,
844  const std::string& loop_name,
845  llvm::Function* parent_func,
846  CgenState* cgen_state) -> llvm::BasicBlock* {
847  // make sure we have quals to hoist
848  bool has_quals_to_hoist = false;
849  for (const auto& qual : hoisted_quals) {
850  // check to see if the filter was previously hoisted. if all filters were
851  // previously hoisted, this callback becomes a noop
852  if (plan_state_->hoisted_filters_.count(qual) == 0) {
853  has_quals_to_hoist = true;
854  break;
855  }
856  }
857 
858  if (!has_quals_to_hoist) {
859  return nullptr;
860  }
861 
862  AUTOMATIC_IR_METADATA(cgen_state);
863 
864  llvm::IRBuilder<>& builder = cgen_state->ir_builder_;
865  auto& context = builder.getContext();
866 
867  const auto filter_bb =
868  llvm::BasicBlock::Create(context,
869  "hoisted_left_join_filters_" + loop_name,
870  parent_func,
871  /*insert_before=*/true_bb);
872  builder.SetInsertPoint(filter_bb);
873 
874  llvm::Value* filter_lv = cgen_state_->llBool(true);
875  CodeGenerator code_generator(this);
876  CHECK(plan_state_);
877  for (const auto& qual : hoisted_quals) {
878  if (plan_state_->hoisted_filters_.insert(qual).second) {
879  // qual was inserted into the hoisted filters map, which means we have not
880  // seen this qual before. Generate filter.
881  VLOG(1) << "Generating code for hoisted left hand side qualifier "
882  << qual->toString();
883  auto cond = code_generator.toBool(
884  code_generator.codegen(qual.get(), true, co).front());
885  filter_lv = builder.CreateAnd(filter_lv, cond);
886  }
887  }
888  CHECK(filter_lv->getType()->isIntegerTy(1));
889 
890  builder.CreateCondBr(filter_lv, true_bb, exit_bb);
891  return filter_bb;
892  };
893  }
894  }
895  }
896  }
897  return nullptr;
898 }
899 
900 std::function<llvm::Value*(const std::vector<llvm::Value*>&, llvm::Value*)>
902  const size_t level_idx,
903  const CompilationOptions& co) {
904  AUTOMATIC_IR_METADATA(cgen_state_.get());
905  if (!co.filter_on_deleted_column) {
906  return nullptr;
907  }
908  CHECK_LT(level_idx + 1, ra_exe_unit.input_descs.size());
909  const auto input_desc = ra_exe_unit.input_descs[level_idx + 1];
910  if (input_desc.getSourceType() != InputSourceType::TABLE) {
911  return nullptr;
912  }
913 
914  const auto deleted_cd = plan_state_->getDeletedColForTable(input_desc.getTableKey());
915  if (!deleted_cd) {
916  return nullptr;
917  }
918  CHECK(deleted_cd->columnType.is_boolean());
919  const auto deleted_expr = makeExpr<Analyzer::ColumnVar>(
920  deleted_cd->columnType,
921  shared::ColumnKey{input_desc.getTableKey(), deleted_cd->columnId},
922  input_desc.getNestLevel());
923  return [this, deleted_expr, level_idx, &co](const std::vector<llvm::Value*>& prev_iters,
924  llvm::Value* have_more_inner_rows) {
925  const auto matching_row_index = addJoinLoopIterator(prev_iters, level_idx + 1);
926  // Avoid fetching the deleted column from a position which is not valid.
927  // An invalid position can be returned by a one to one hash lookup (negative)
928  // or at the end of iteration over a set of matching values.
929  llvm::Value* is_valid_it{nullptr};
930  if (have_more_inner_rows) {
931  is_valid_it = have_more_inner_rows;
932  } else {
933  is_valid_it = cgen_state_->ir_builder_.CreateICmp(
934  llvm::ICmpInst::ICMP_SGE, matching_row_index, cgen_state_->llInt<int64_t>(0));
935  }
936  const auto it_valid_bb = llvm::BasicBlock::Create(
937  cgen_state_->context_, "it_valid", cgen_state_->current_func_);
938  const auto it_not_valid_bb = llvm::BasicBlock::Create(
939  cgen_state_->context_, "it_not_valid", cgen_state_->current_func_);
940  cgen_state_->ir_builder_.CreateCondBr(is_valid_it, it_valid_bb, it_not_valid_bb);
941  const auto row_is_deleted_bb = llvm::BasicBlock::Create(
942  cgen_state_->context_, "row_is_deleted", cgen_state_->current_func_);
943  cgen_state_->ir_builder_.SetInsertPoint(it_valid_bb);
944  CodeGenerator code_generator(this);
945  const auto row_is_deleted = code_generator.toBool(
946  code_generator.codegen(deleted_expr.get(), true, co).front());
947  cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
948  cgen_state_->ir_builder_.SetInsertPoint(it_not_valid_bb);
949  const auto row_is_deleted_default = cgen_state_->llBool(false);
950  cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
951  cgen_state_->ir_builder_.SetInsertPoint(row_is_deleted_bb);
952  auto row_is_deleted_or_default =
953  cgen_state_->ir_builder_.CreatePHI(row_is_deleted->getType(), 2);
954  row_is_deleted_or_default->addIncoming(row_is_deleted, it_valid_bb);
955  row_is_deleted_or_default->addIncoming(row_is_deleted_default, it_not_valid_bb);
956  return row_is_deleted_or_default;
957  };
958 }
959 
960 std::shared_ptr<HashJoin> Executor::buildCurrentLevelHashTable(
961  const JoinCondition& current_level_join_conditions,
962  size_t level_idx,
963  RelAlgExecutionUnit& ra_exe_unit,
964  const CompilationOptions& co,
965  const std::vector<InputTableInfo>& query_infos,
966  ColumnCacheMap& column_cache,
967  std::vector<std::string>& fail_reasons) {
968  AUTOMATIC_IR_METADATA(cgen_state_.get());
969  std::shared_ptr<HashJoin> current_level_hash_table;
970  auto handleNonHashtableQual = [&ra_exe_unit, &level_idx, this](
971  JoinType join_type,
972  std::shared_ptr<Analyzer::Expr> qual) {
973  if (join_type == JoinType::LEFT) {
974  plan_state_->addNonHashtableQualForLeftJoin(level_idx, qual);
975  } else {
976  add_qualifier_to_execution_unit(ra_exe_unit, qual);
977  }
978  };
979  for (const auto& join_qual : current_level_join_conditions.quals) {
980  auto qual_bin_oper = std::dynamic_pointer_cast<Analyzer::BinOper>(join_qual);
981  if (current_level_hash_table || !qual_bin_oper ||
982  !IS_EQUIVALENCE(qual_bin_oper->get_optype())) {
983  handleNonHashtableQual(current_level_join_conditions.type, join_qual);
984  if (!current_level_hash_table) {
985  fail_reasons.emplace_back("No equijoin expression found");
986  }
987  continue;
988  }
989  check_valid_join_qual(qual_bin_oper);
990  JoinHashTableOrError hash_table_or_error;
991  if (!current_level_hash_table) {
992  hash_table_or_error = buildHashTableForQualifier(
993  qual_bin_oper,
994  query_infos,
997  current_level_join_conditions.type,
999  column_cache,
1000  ra_exe_unit.hash_table_build_plan_dag,
1001  ra_exe_unit.query_hint,
1002  ra_exe_unit.table_id_to_node_map);
1003  current_level_hash_table = hash_table_or_error.hash_table;
1004  }
1005  if (hash_table_or_error.hash_table) {
1006  plan_state_->join_info_.join_hash_tables_.push_back(hash_table_or_error.hash_table);
1007  plan_state_->join_info_.equi_join_tautologies_.push_back(qual_bin_oper);
1008  } else {
1009  fail_reasons.push_back(hash_table_or_error.fail_reason);
1010  if (!current_level_hash_table) {
1011  VLOG(2) << "Building a hashtable based on a qual " << qual_bin_oper->toString()
1012  << " fails: " << hash_table_or_error.fail_reason;
1013  }
1014  handleNonHashtableQual(current_level_join_conditions.type, qual_bin_oper);
1015  }
1016  }
1017  return current_level_hash_table;
1018 }
1019 
1021  if (!cgen_state_->filter_func_) {
1022  return;
1023  }
1024 
1025  // Loop over all the instructions used in the filter func.
1026  // The filter func instructions were generated as if for row func.
1027  // Remap any values used by those instructions to filter func args
1028  // and remember to forward them through the call in the row func.
1029  for (auto bb_it = cgen_state_->filter_func_->begin();
1030  bb_it != cgen_state_->filter_func_->end();
1031  ++bb_it) {
1032  for (auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1033  size_t i = 0;
1034  for (auto op_it = instr_it->value_op_begin(); op_it != instr_it->value_op_end();
1035  ++op_it, ++i) {
1036  llvm::Value* v = *op_it;
1037 
1038  // The last LLVM operand on a call instruction is the function to be called. Never
1039  // remap it.
1040  if (llvm::dyn_cast<const llvm::CallInst>(instr_it) &&
1041  op_it == instr_it->value_op_end() - 1) {
1042  continue;
1043  }
1044 
1045  CHECK(v);
1046  if (auto* instr = llvm::dyn_cast<llvm::Instruction>(v);
1047  instr && instr->getParent() &&
1048  instr->getParent()->getParent() == cgen_state_->row_func_) {
1049  // Remember that this filter func arg is needed.
1050  cgen_state_->filter_func_args_[v] = nullptr;
1051  } else if (auto* argum = llvm::dyn_cast<llvm::Argument>(v);
1052  argum && argum->getParent() == cgen_state_->row_func_) {
1053  // Remember that this filter func arg is needed.
1054  cgen_state_->filter_func_args_[v] = nullptr;
1055  }
1056  }
1057  }
1058  }
1059 
1060  // Create filter_func2 with parameters only for those row func values that are known to
1061  // be used in the filter func code.
1062  std::vector<llvm::Type*> filter_func_arg_types;
1063  filter_func_arg_types.reserve(cgen_state_->filter_func_args_.v_.size());
1064  for (auto& arg : cgen_state_->filter_func_args_.v_) {
1065  filter_func_arg_types.push_back(arg->getType());
1066  }
1067  auto ft = llvm::FunctionType::get(
1068  get_int_type(32, cgen_state_->context_), filter_func_arg_types, false);
1069  cgen_state_->filter_func_->setName("old_filter_func");
1070  auto filter_func2 = llvm::Function::Create(ft,
1071  llvm::Function::ExternalLinkage,
1072  "filter_func",
1073  cgen_state_->filter_func_->getParent());
1074  CHECK_EQ(filter_func2->arg_size(), cgen_state_->filter_func_args_.v_.size());
1075  auto arg_it = cgen_state_->filter_func_args_.begin();
1076  size_t i = 0;
1077  for (llvm::Function::arg_iterator I = filter_func2->arg_begin(),
1078  E = filter_func2->arg_end();
1079  I != E;
1080  ++I, ++arg_it) {
1081  arg_it->second = &*I;
1082  if (arg_it->first->hasName()) {
1083  I->setName(arg_it->first->getName());
1084  } else {
1085  I->setName("extra" + std::to_string(i++));
1086  }
1087  }
1088 
1089  // copy the filter_func function body over
1090  // see
1091  // https://stackoverflow.com/questions/12864106/move-function-body-avoiding-full-cloning/18751365
1092  filter_func2->getBasicBlockList().splice(
1093  filter_func2->begin(), cgen_state_->filter_func_->getBasicBlockList());
1094 
1095  if (cgen_state_->current_func_ == cgen_state_->filter_func_) {
1096  cgen_state_->current_func_ = filter_func2;
1097  }
1098  cgen_state_->filter_func_ = filter_func2;
1099 
1100  // loop over all the operands in the filter func
1101  for (auto bb_it = cgen_state_->filter_func_->begin();
1102  bb_it != cgen_state_->filter_func_->end();
1103  ++bb_it) {
1104  for (auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1105  size_t i = 0;
1106  for (auto op_it = instr_it->op_begin(); op_it != instr_it->op_end(); ++op_it, ++i) {
1107  llvm::Value* v = op_it->get();
1108  if (auto arg_it = cgen_state_->filter_func_args_.find(v);
1109  arg_it != cgen_state_->filter_func_args_.end()) {
1110  // replace row func value with a filter func arg
1111  llvm::Use* use = &*op_it;
1112  use->set(arg_it->second);
1113  }
1114  }
1115  }
1116  }
1117 }
1118 
1119 llvm::Value* Executor::addJoinLoopIterator(const std::vector<llvm::Value*>& prev_iters,
1120  const size_t level_idx) {
1121  AUTOMATIC_IR_METADATA(cgen_state_.get());
1122  // Iterators are added for loop-outer joins when the head of the loop is generated,
1123  // then once again when the body if generated. Allow this instead of special handling
1124  // of call sites.
1125  const auto it = cgen_state_->scan_idx_to_hash_pos_.find(level_idx);
1126  if (it != cgen_state_->scan_idx_to_hash_pos_.end()) {
1127  return it->second;
1128  }
1129  CHECK(!prev_iters.empty());
1130  llvm::Value* matching_row_index = prev_iters.back();
1131  const auto it_ok =
1132  cgen_state_->scan_idx_to_hash_pos_.emplace(level_idx, matching_row_index);
1133  CHECK(it_ok.second);
1134  return matching_row_index;
1135 }
1136 
1137 void Executor::codegenJoinLoops(const std::vector<JoinLoop>& join_loops,
1138  const RelAlgExecutionUnit& ra_exe_unit,
1139  GroupByAndAggregate& group_by_and_aggregate,
1140  llvm::Function* query_func,
1141  llvm::BasicBlock* entry_bb,
1143  const CompilationOptions& co,
1144  const ExecutionOptions& eo) {
1145  AUTOMATIC_IR_METADATA(cgen_state_.get());
1146  const auto exit_bb =
1147  llvm::BasicBlock::Create(cgen_state_->context_, "exit", cgen_state_->current_func_);
1148  cgen_state_->ir_builder_.SetInsertPoint(exit_bb);
1149  cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
1150  cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1151  CodeGenerator code_generator(this);
1152 
1153  llvm::BasicBlock* loops_entry_bb{nullptr};
1154  auto has_range_join =
1155  std::any_of(join_loops.begin(), join_loops.end(), [](const auto& join_loop) {
1156  return join_loop.kind() == JoinLoopKind::MultiSet;
1157  });
1158  if (has_range_join) {
1159  CHECK_EQ(join_loops.size(), size_t(1));
1160  const auto element_count =
1161  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_), 9);
1162 
1163  auto compute_packed_offset = [](const int32_t x, const int32_t y) -> uint64_t {
1164  const uint64_t y_shifted = static_cast<uint64_t>(y) << 32;
1165  return y_shifted | static_cast<uint32_t>(x);
1166  };
1167 
1168  const auto values_arr = std::vector<llvm::Constant*>{
1169  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_), 0),
1170  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1171  compute_packed_offset(0, 1)),
1172  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1173  compute_packed_offset(0, -1)),
1174  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1175  compute_packed_offset(1, 0)),
1176  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1177  compute_packed_offset(1, 1)),
1178  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1179  compute_packed_offset(1, -1)),
1180  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1181  compute_packed_offset(-1, 0)),
1182  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1183  compute_packed_offset(-1, 1)),
1184  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1185  compute_packed_offset(-1, -1))};
1186 
1187  const auto constant_values_array = llvm::ConstantArray::get(
1188  get_int_array_type(64, 9, cgen_state_->context_), values_arr);
1189  CHECK(cgen_state_->module_);
1190  const auto values =
1191  new llvm::GlobalVariable(*cgen_state_->module_,
1192  get_int_array_type(64, 9, cgen_state_->context_),
1193  true,
1194  llvm::GlobalValue::LinkageTypes::InternalLinkage,
1195  constant_values_array);
1196  JoinLoop join_loop(
1199  [element_count, values](const std::vector<llvm::Value*>& v) {
1200  JoinLoopDomain domain{{0}};
1201  domain.element_count = element_count;
1202  domain.values_buffer = values;
1203  return domain;
1204  },
1205  nullptr,
1206  nullptr,
1207  nullptr,
1208  nullptr,
1209  "range_key_loop");
1210 
1211  loops_entry_bb = JoinLoop::codegen(
1212  {join_loop},
1213  [this,
1214  query_func,
1215  &query_mem_desc,
1216  &co,
1217  &eo,
1218  &group_by_and_aggregate,
1219  &join_loops,
1220  &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
1221  auto& builder = cgen_state_->ir_builder_;
1222 
1223  auto body_exit_bb =
1224  llvm::BasicBlock::Create(cgen_state_->context_,
1225  "range_key_inner_body_exit",
1226  builder.GetInsertBlock()->getParent());
1227 
1228  auto range_key_body_bb =
1229  llvm::BasicBlock::Create(cgen_state_->context_,
1230  "range_key_loop_body",
1231  builder.GetInsertBlock()->getParent());
1232  builder.SetInsertPoint(range_key_body_bb);
1233 
1234  const auto body_loops_entry_bb = JoinLoop::codegen(
1235  join_loops,
1236  [this,
1237  query_func,
1238  &query_mem_desc,
1239  &co,
1240  &eo,
1241  &group_by_and_aggregate,
1242  &join_loops,
1243  &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
1244  addJoinLoopIterator(prev_iters, join_loops.size());
1245  auto& builder = cgen_state_->ir_builder_;
1246  const auto loop_body_bb =
1247  llvm::BasicBlock::Create(builder.getContext(),
1248  "loop_body",
1249  builder.GetInsertBlock()->getParent());
1250  builder.SetInsertPoint(loop_body_bb);
1251  const bool can_return_error =
1252  compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1253  if (can_return_error || cgen_state_->needs_error_check_ ||
1254  eo.with_dynamic_watchdog || eo.allow_runtime_query_interrupt) {
1255  createErrorCheckControlFlow(query_func,
1256  eo.with_dynamic_watchdog,
1257  eo.allow_runtime_query_interrupt,
1258  join_loops,
1259  co.device_type,
1260  group_by_and_aggregate.query_infos_);
1261  }
1262  return loop_body_bb;
1263  },
1264  prev_iters.back(),
1265  body_exit_bb,
1266  cgen_state_.get());
1267 
1268  builder.SetInsertPoint(range_key_body_bb);
1269  cgen_state_->ir_builder_.CreateBr(body_loops_entry_bb);
1270 
1271  builder.SetInsertPoint(body_exit_bb);
1272  return range_key_body_bb;
1273  },
1274  code_generator.posArg(nullptr),
1275  exit_bb,
1276  cgen_state_.get());
1277  } else {
1278  loops_entry_bb = JoinLoop::codegen(
1279  join_loops,
1280  /*body_codegen=*/
1281  [this,
1282  query_func,
1283  &query_mem_desc,
1284  &co,
1285  &eo,
1286  &group_by_and_aggregate,
1287  &join_loops,
1288  &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
1289  AUTOMATIC_IR_METADATA(cgen_state_.get());
1290  addJoinLoopIterator(prev_iters, join_loops.size());
1291  auto& builder = cgen_state_->ir_builder_;
1292  const auto loop_body_bb = llvm::BasicBlock::Create(
1293  builder.getContext(), "loop_body", builder.GetInsertBlock()->getParent());
1294  builder.SetInsertPoint(loop_body_bb);
1295  const bool can_return_error =
1296  compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1297  if (can_return_error || cgen_state_->needs_error_check_ ||
1299  createErrorCheckControlFlow(query_func,
1302  join_loops,
1303  co.device_type,
1304  group_by_and_aggregate.query_infos_);
1305  }
1306  return loop_body_bb;
1307  },
1308  /*outer_iter=*/code_generator.posArg(nullptr),
1309  exit_bb,
1310  cgen_state_.get());
1311  }
1312  CHECK(loops_entry_bb);
1313  cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1314  cgen_state_->ir_builder_.CreateBr(loops_entry_bb);
1315 }
1316 
1318  Analyzer::Expr* group_by_col,
1319  const size_t col_width,
1320  const CompilationOptions& co,
1321  const bool translate_null_val,
1322  const int64_t translated_null_val,
1323  DiamondCodegen& diamond_codegen,
1324  std::stack<llvm::BasicBlock*>& array_loops,
1325  const bool thread_mem_shared) {
1326  AUTOMATIC_IR_METADATA(cgen_state_.get());
1327  CHECK_GE(col_width, sizeof(int32_t));
1328  CodeGenerator code_generator(this);
1329  auto group_key = code_generator.codegen(group_by_col, true, co).front();
1330  auto key_to_cache = group_key;
1331  if (dynamic_cast<Analyzer::UOper*>(group_by_col) &&
1332  static_cast<Analyzer::UOper*>(group_by_col)->get_optype() == kUNNEST) {
1333  auto preheader = cgen_state_->ir_builder_.GetInsertBlock();
1334  auto array_loop_head = llvm::BasicBlock::Create(cgen_state_->context_,
1335  "array_loop_head",
1336  cgen_state_->current_func_,
1337  preheader->getNextNode());
1338  diamond_codegen.setFalseTarget(array_loop_head);
1339  const auto ret_ty = get_int_type(32, cgen_state_->context_);
1340  auto array_idx_ptr = cgen_state_->ir_builder_.CreateAlloca(ret_ty);
1341  CHECK(array_idx_ptr);
1342  cgen_state_->ir_builder_.CreateStore(cgen_state_->llInt(int32_t(0)), array_idx_ptr);
1343  const auto arr_expr = static_cast<Analyzer::UOper*>(group_by_col)->get_operand();
1344  const auto& array_ti = arr_expr->get_type_info();
1345  CHECK(array_ti.is_array());
1346  const auto& elem_ti = array_ti.get_elem_type();
1347  auto array_len =
1348  (array_ti.get_size() > 0)
1349  ? cgen_state_->llInt(array_ti.get_size() / elem_ti.get_size())
1350  : cgen_state_->emitExternalCall(
1351  "array_size",
1352  ret_ty,
1353  {group_key,
1354  code_generator.posArg(arr_expr),
1355  cgen_state_->llInt(log2_bytes(elem_ti.get_logical_size()))});
1356  cgen_state_->ir_builder_.CreateBr(array_loop_head);
1357  cgen_state_->ir_builder_.SetInsertPoint(array_loop_head);
1358  CHECK(array_len);
1359  auto array_idx = cgen_state_->ir_builder_.CreateLoad(
1360  array_idx_ptr->getType()->getPointerElementType(), array_idx_ptr);
1361  auto bound_check = cgen_state_->ir_builder_.CreateICmp(
1362  llvm::ICmpInst::ICMP_SLT, array_idx, array_len);
1363  auto array_loop_body = llvm::BasicBlock::Create(
1364  cgen_state_->context_, "array_loop_body", cgen_state_->current_func_);
1365  cgen_state_->ir_builder_.CreateCondBr(
1366  bound_check,
1367  array_loop_body,
1368  array_loops.empty() ? diamond_codegen.orig_cond_false_ : array_loops.top());
1369  cgen_state_->ir_builder_.SetInsertPoint(array_loop_body);
1370  cgen_state_->ir_builder_.CreateStore(
1371  cgen_state_->ir_builder_.CreateAdd(array_idx, cgen_state_->llInt(int32_t(1))),
1372  array_idx_ptr);
1373  auto array_at_fname = "array_at_" + numeric_type_name(elem_ti);
1374  if (array_ti.get_size() < 0) {
1375  if (array_ti.get_notnull()) {
1376  array_at_fname = "notnull_" + array_at_fname;
1377  }
1378  array_at_fname = "varlen_" + array_at_fname;
1379  }
1380  const auto ar_ret_ty =
1381  elem_ti.is_fp()
1382  ? (elem_ti.get_type() == kDOUBLE
1383  ? llvm::Type::getDoubleTy(cgen_state_->context_)
1384  : llvm::Type::getFloatTy(cgen_state_->context_))
1385  : get_int_type(elem_ti.get_logical_size() * 8, cgen_state_->context_);
1386  group_key = cgen_state_->emitExternalCall(
1387  array_at_fname,
1388  ar_ret_ty,
1389  {group_key, code_generator.posArg(arr_expr), array_idx});
1391  elem_ti, isArchMaxwell(co.device_type), thread_mem_shared)) {
1392  key_to_cache = spillDoubleElement(group_key, ar_ret_ty);
1393  } else {
1394  key_to_cache = group_key;
1395  }
1396  CHECK(array_loop_head);
1397  array_loops.push(array_loop_head);
1398  }
1399  cgen_state_->group_by_expr_cache_.push_back(key_to_cache);
1400  llvm::Value* orig_group_key{nullptr};
1401  if (translate_null_val) {
1402  const std::string translator_func_name(
1403  col_width == sizeof(int32_t) ? "translate_null_key_i32_" : "translate_null_key_");
1404  const auto& ti = group_by_col->get_type_info();
1405  const auto key_type = get_int_type(ti.get_logical_size() * 8, cgen_state_->context_);
1406  orig_group_key = group_key;
1407  group_key = cgen_state_->emitCall(
1408  translator_func_name + numeric_type_name(ti),
1409  {group_key,
1410  static_cast<llvm::Value*>(
1411  llvm::ConstantInt::get(key_type, inline_int_null_val(ti))),
1412  static_cast<llvm::Value*>(llvm::ConstantInt::get(
1413  llvm::Type::getInt64Ty(cgen_state_->context_), translated_null_val))});
1414  }
1415  group_key = cgen_state_->ir_builder_.CreateBitCast(
1416  cgen_state_->castToTypeIn(group_key, col_width * 8),
1417  get_int_type(col_width * 8, cgen_state_->context_));
1418  if (orig_group_key) {
1419  orig_group_key = cgen_state_->ir_builder_.CreateBitCast(
1420  cgen_state_->castToTypeIn(orig_group_key, col_width * 8),
1421  get_int_type(col_width * 8, cgen_state_->context_));
1422  }
1423  return {group_key, orig_group_key};
1424 }
1425 
1427  Executor* executor,
1428  llvm::Value* nullable_lv,
1429  const SQLTypeInfo& nullable_ti,
1430  const std::string& name)
1431  : cgen_state(cgen_state), name(name) {
1432  AUTOMATIC_IR_METADATA(cgen_state);
1433  CHECK(nullable_ti.is_number() || nullable_ti.is_time() || nullable_ti.is_boolean() ||
1434  nullable_ti.is_dict_encoded_string());
1435 
1436  llvm::Value* is_null_lv{nullptr};
1437  if (nullable_ti.is_fp()) {
1438  is_null_lv = cgen_state->ir_builder_.CreateFCmp(
1439  llvm::FCmpInst::FCMP_OEQ, nullable_lv, cgen_state->inlineFpNull(nullable_ti));
1440  } else if (nullable_ti.is_boolean() &&
1441  nullable_lv->getType()->getIntegerBitWidth() == 1) {
1442  is_null_lv = cgen_state->ir_builder_.CreateICmp(
1443  llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->llBool(true));
1444  } else {
1445  is_null_lv = cgen_state->ir_builder_.CreateICmp(
1446  llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->inlineIntNull(nullable_ti));
1447  }
1448  CHECK(is_null_lv);
1449  null_check =
1450  std::make_unique<DiamondCodegen>(is_null_lv, executor, false, name, nullptr, false);
1451 
1452  // generate a phi node depending on whether we got a null or not
1453  nullcheck_bb = llvm::BasicBlock::Create(
1454  cgen_state->context_, name + "_bb", cgen_state->current_func_);
1455 
1456  // update the blocks created by diamond codegen to point to the newly created phi
1457  // block
1458  cgen_state->ir_builder_.SetInsertPoint(null_check->cond_true_);
1459  cgen_state->ir_builder_.CreateBr(nullcheck_bb);
1460  cgen_state->ir_builder_.SetInsertPoint(null_check->cond_false_);
1461 }
1462 
1463 llvm::Value* CodeGenerator::NullCheckCodegen::finalize(llvm::Value* null_lv,
1464  llvm::Value* notnull_lv) {
1465  AUTOMATIC_IR_METADATA(cgen_state);
1466  CHECK(null_check);
1467  cgen_state->ir_builder_.CreateBr(nullcheck_bb);
1468 
1469  CHECK_EQ(null_lv->getType(), notnull_lv->getType());
1470 
1471  cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1472  nullcheck_value =
1473  cgen_state->ir_builder_.CreatePHI(null_lv->getType(), 2, name + "_value");
1474  nullcheck_value->addIncoming(notnull_lv, null_check->cond_false_);
1475  nullcheck_value->addIncoming(null_lv, null_check->cond_true_);
1476 
1477  null_check.reset(nullptr);
1478  cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1479  return nullcheck_value;
1480 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
bool g_enable_left_join_filter_hoisting
Definition: Execute.cpp:100
NullCheckCodegen(CgenState *cgen_state, Executor *executor, llvm::Value *nullable_lv, const SQLTypeInfo &nullable_ti, const std::string &name="")
Definition: IRCodegen.cpp:1426
void codegenJoinLoops(const std::vector< JoinLoop > &join_loops, const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, llvm::Function *query_func, llvm::BasicBlock *entry_bb, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const ExecutionOptions &eo)
Definition: IRCodegen.cpp:1137
#define IS_LOGIC(X)
Definition: sqldefs.h:61
const Expr * get_partition_count() const
Definition: Analyzer.h:1049
#define NULL_DOUBLE
JoinType
Definition: sqldefs.h:165
std::vector< llvm::Value * > outer_join_match_found_per_level_
Definition: CgenState.h:388
bool is_constant_expr() const
Definition: Analyzer.h:1124
std::unordered_map< size_t, std::vector< std::shared_ptr< Analyzer::Expr > > > left_join_non_hashtable_quals_
Definition: PlanState.h:63
llvm::Value * codegenConstantWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
Definition: IRCodegen.cpp:303
llvm::BasicBlock * nullcheck_bb
llvm::Value * element_count
Definition: JoinLoop.h:46
llvm::Value * values_buffer
Definition: JoinLoop.h:49
#define IS_EQUIVALENCE(X)
Definition: sqldefs.h:69
llvm::Value * codegenArith(const Analyzer::BinOper *, const CompilationOptions &)
CgenState * cgen_state_
GroupColLLVMValue groupByColumnCodegen(Analyzer::Expr *group_by_col, const size_t col_width, const CompilationOptions &, const bool translate_null_val, const int64_t translated_null_val, DiamondCodegen &, std::stack< llvm::BasicBlock * > &, const bool thread_mem_shared)
Definition: IRCodegen.cpp:1317
bool is_fp() const
Definition: sqltypes.h:584
llvm::IRBuilder ir_builder_
Definition: CgenState.h:377
std::function< llvm::BasicBlock *(llvm::BasicBlock *, llvm::BasicBlock *, const std::string &, llvm::Function *, CgenState *)> HoistedFiltersCallback
Definition: JoinLoop.h:61
llvm::Value * posArg(const Analyzer::Expr *) const
Definition: ColumnIR.cpp:580
std::string join(T const &container, std::string const &delim)
std::vector< InputDescriptor > input_descs
#define UNREACHABLE()
Definition: Logger.h:337
#define CHECK_GE(x, y)
Definition: Logger.h:306
bool need_patch_unnest_double(const SQLTypeInfo &ti, const bool is_maxwell, const bool mem_shared)
Definition: sqldefs.h:48
llvm::ConstantInt * llBool(const bool v) const
Definition: CgenState.h:256
virtual std::vector< llvm::Value * > codegenColumn(const Analyzer::ColumnVar *, const bool fetch_column, const CompilationOptions &)
Definition: ColumnIR.cpp:93
void set_constant_expr() const
Definition: Analyzer.h:1123
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:89
llvm::Value * codegenArrayAt(const Analyzer::BinOper *, const CompilationOptions &)
Definition: ArrayIR.cpp:26
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:381
void setFalseTarget(llvm::BasicBlock *cond_false)
QualsConjunctiveForm qual_to_conjunctive_form(const std::shared_ptr< Analyzer::Expr > qual_expr)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
bool is_number() const
Definition: sqltypes.h:585
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::vector< llvm::Value * > codegenGeoBinOper(const Analyzer::GeoBinOper *, const CompilationOptions &)
Definition: GeoIR.cpp:242
bool is_time() const
Definition: sqltypes.h:586
std::shared_ptr< HashJoin > hash_table
Definition: Execute.h:1083
double get_bound_val(const Analyzer::Expr *bound_expr) const
Definition: Analyzer.cpp:3752
std::string to_string(char const *&&v)
llvm::Function * row_func_
Definition: CgenState.h:367
llvm::Value * codegenIsNull(const Analyzer::UOper *, const CompilationOptions &)
Definition: LogicalIR.cpp:380
SQLOps get_optype() const
Definition: Analyzer.h:452
std::vector< llvm::Value * > codegenGeoExpr(const Analyzer::GeoExpr *, const CompilationOptions &)
Definition: GeoIR.cpp:95
llvm::LLVMContext & context_
Definition: CgenState.h:375
llvm::Function * current_func_
Definition: CgenState.h:369
llvm::Value * get_arg_by_name(llvm::Function *func, const std::string &name)
Definition: Execute.h:167
std::set< shared::TableKey > visitFunctionOper(const Analyzer::FunctionOper *func_expr) const final
Definition: IRCodegen.cpp:760
#define INJECT_TIMER(DESC)
Definition: measure.h:93
#define CHECK_NE(x, y)
Definition: Logger.h:302
const JoinQualsPerNestingLevel join_quals
std::vector< llvm::Value * > codegenGeoUOper(const Analyzer::GeoUOper *, const CompilationOptions &)
Definition: GeoIR.cpp:162
llvm::ConstantInt * inlineIntNull(const SQLTypeInfo &)
Definition: CgenState.cpp:64
TableIdToNodeMap table_id_to_node_map
llvm::Value * codegenWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
Definition: IRCodegen.cpp:376
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
Definition: CastIR.cpp:427
llvm::Value * codegenFunctionOper(const Analyzer::FunctionOper *, const CompilationOptions &)
Executor * executor_
static llvm::BasicBlock * codegen(const std::vector< JoinLoop > &join_loops, const std::function< llvm::BasicBlock *(const std::vector< llvm::Value * > &)> &body_codegen, llvm::Value *outer_iter, llvm::BasicBlock *exit_bb, CgenState *cgen_state)
Definition: JoinLoop.cpp:50
void add_qualifier_to_execution_unit(RelAlgExecutionUnit &ra_exe_unit, const std::shared_ptr< Analyzer::Expr > &qual)
Definition: IRCodegen.cpp:474
const std::vector< InputTableInfo > & query_infos_
Definition: PlanState.h:67
bool needs_error_check_
Definition: CgenState.h:397
bool is_boolean() const
Definition: sqltypes.h:587
ExpressionRange getExpressionRange(const Analyzer::BinOper *expr, const std::vector< InputTableInfo > &query_infos, const Executor *, boost::optional< std::list< std::shared_ptr< Analyzer::Expr >>> simple_quals)
std::vector< llvm::Value * > codegenArrayExpr(const Analyzer::ArrayExpr *, const CompilationOptions &)
Definition: ArrayIR.cpp:91
#define AUTOMATIC_IR_METADATA(CGENSTATE)
llvm::BasicBlock * orig_cond_false_
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:79
llvm::Value * codegenUMinus(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
Definition: CgenState.cpp:216
llvm::Value * slot_lookup_result
Definition: JoinLoop.h:47
ExecutorDeviceType device_type
Definition: sqldefs.h:33
PlanState * plan_state_
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
Definition: IRCodegen.cpp:30
#define CHECK_LT(x, y)
Definition: Logger.h:303
const std::vector< InputTableInfo > & query_infos_
Definition: sqldefs.h:71
Expression class for string functions The &quot;arg&quot; constructor parameter must be an expression that reso...
Definition: Analyzer.h:1479
std::set< shared::TableKey > aggregateResult(const std::set< shared::TableKey > &aggregate, const std::set< shared::TableKey > &next_result) const final
Definition: IRCodegen.cpp:779
std::shared_ptr< HashJoin > buildCurrentLevelHashTable(const JoinCondition &current_level_join_conditions, size_t level_idx, RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache, std::vector< std::string > &fail_reasons)
Definition: IRCodegen.cpp:960
#define IS_ARITHMETIC(X)
Definition: sqldefs.h:62
bool isHintRegistered(const QueryHint hint) const
Definition: QueryHint.h:348
const Expr * get_arg() const
Definition: Analyzer.h:810
std::set< shared::TableKey > visitBinOper(const Analyzer::BinOper *bin_oper) const final
Definition: IRCodegen.cpp:769
std::unordered_map< shared::TableKey, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
std::set< shared::TableKey > visitUOper(const Analyzer::UOper *u_oper) const final
Definition: IRCodegen.cpp:775
int32_t get_partition_count_val() const
Definition: Analyzer.cpp:3762
static const int32_t ERR_WIDTH_BUCKET_INVALID_ARGUMENT
Definition: Execute.h:1443
llvm::Value * toBool(llvm::Value *)
Definition: LogicalIR.cpp:343
std::vector< llvm::Value * > codegenGeoColumnVar(const Analyzer::GeoColumnVar *, const bool fetch_columns, const CompilationOptions &co)
Definition: GeoIR.cpp:52
llvm::Value * codegenFunctionOperWithCustomTypeHandling(const Analyzer::FunctionOperWithCustomTypeHandling *, const CompilationOptions &)
llvm::Value * codegenCmp(const Analyzer::BinOper *, const CompilationOptions &)
Definition: CompareIR.cpp:230
std::list< std::shared_ptr< Analyzer::Expr > > getSimpleQuals() const
Definition: PlanState.h:98
const Expr * get_target_value() const
Definition: Analyzer.h:1046
std::list< std::shared_ptr< Analyzer::Expr > > quals
llvm::ConstantInt * llInt(const T v) const
Definition: CgenState.h:242
llvm::Value * codegenUnnest(const Analyzer::UOper *, const CompilationOptions &)
Definition: ArrayIR.cpp:20
llvm::Value * addJoinLoopIterator(const std::vector< llvm::Value * > &prev_iters, const size_t level_idx)
Definition: IRCodegen.cpp:1119
std::list< std::shared_ptr< Analyzer::Expr > > quals
RegisteredQueryHint query_hint
llvm::Value * finalize(llvm::Value *null_lv, llvm::Value *notnull_lv)
Definition: IRCodegen.cpp:1463
#define CHECK(condition)
Definition: Logger.h:291
bool can_skip_out_of_bound_check() const
Definition: Analyzer.h:1121
llvm::Value * codegenLogical(const Analyzer::BinOper *, const CompilationOptions &)
Definition: LogicalIR.cpp:298
void check_if_loop_join_is_allowed(RelAlgExecutionUnit &ra_exe_unit, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, const size_t level_idx, const std::string &fail_reason)
Definition: IRCodegen.cpp:484
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
llvm::ConstantInt * ll_bool(const bool v, llvm::LLVMContext &context)
size_t loop_join_inner_table_max_num_rows
Definition: QueryHint.h:324
llvm::Value * codegenCast(const Analyzer::UOper *, const CompilationOptions &)
Definition: CastIR.cpp:21
uint32_t log2_bytes(const uint32_t bytes)
Definition: Execute.h:177
std::string numeric_type_name(const SQLTypeInfo &ti)
Definition: Execute.h:209
bool is_dict_encoded_string() const
Definition: sqltypes.h:632
Definition: sqltypes.h:62
void skip_out_of_bound_check() const
Definition: Analyzer.h:1122
void redeclareFilterFunction()
Definition: IRCodegen.cpp:1020
bool any_of(std::vector< Analyzer::Expr * > const &target_exprs)
const Expr * get_lower_bound() const
Definition: Analyzer.h:1047
std::vector< JoinLoop > buildJoinLoops(RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache)
Definition: IRCodegen.cpp:544
std::function< llvm::Value *(const std::vector< llvm::Value * > &, llvm::Value *)> buildIsDeletedCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const CompilationOptions &co)
Definition: IRCodegen.cpp:901
string name
Definition: setup.in.py:72
JoinLoop::HoistedFiltersCallback buildHoistLeftHandSideFiltersCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const shared::TableKey &inner_table_key, const CompilationOptions &co)
Definition: IRCodegen.cpp:792
const Expr * get_upper_bound() const
Definition: Analyzer.h:1048
Definition: Datum.h:67
llvm::ArrayType * get_int_array_type(int const width, int count, llvm::LLVMContext &context)
Definition: sqldefs.h:38
SQLOps get_optype() const
Definition: Analyzer.h:383
#define VLOG(n)
Definition: Logger.h:387
std::unique_ptr< DiamondCodegen > null_check
std::set< shared::TableKey > visitColumnVar(const Analyzer::ColumnVar *col_expr) const final
Definition: IRCodegen.cpp:755
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals
#define IS_COMPARISON(X)
Definition: sqldefs.h:58
double doubleval
Definition: Datum.h:74
HashTableBuildDagMap hash_table_build_plan_dag
llvm::ConstantFP * inlineFpNull(const SQLTypeInfo &)
Definition: CgenState.cpp:103
void check_valid_join_qual(std::shared_ptr< Analyzer::BinOper > &bin_oper)
Definition: IRCodegen.cpp:525
Executor * executor() const
size_t get_loop_join_size(const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
Definition: Execute.cpp:1561
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr(const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count)