OmniSciDB  f17484ade4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
IRCodegen.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "../Parser/ParserNode.h"
18 #include "CodeGenerator.h"
19 #include "Execute.h"
20 #include "ExternalExecutor.h"
21 #include "MaxwellCodegenPatch.h"
22 #include "RelAlgTranslator.h"
23 
25 
26 // Driver methods for the IR generation.
27 
29 
30 std::vector<llvm::Value*> CodeGenerator::codegen(const Analyzer::Expr* expr,
31  const bool fetch_columns,
32  const CompilationOptions& co) {
34  if (!expr) {
35  return {posArg(expr)};
36  }
37  auto bin_oper = dynamic_cast<const Analyzer::BinOper*>(expr);
38  if (bin_oper) {
39  return {codegen(bin_oper, co)};
40  }
41  auto u_oper = dynamic_cast<const Analyzer::UOper*>(expr);
42  if (u_oper) {
43  return {codegen(u_oper, co)};
44  }
45  auto geo_col_var = dynamic_cast<const Analyzer::GeoColumnVar*>(expr);
46  if (geo_col_var) {
47  // inherits from ColumnVar, so it is important we check this first
48  return codegenGeoColumnVar(geo_col_var, fetch_columns, co);
49  }
50  auto col_var = dynamic_cast<const Analyzer::ColumnVar*>(expr);
51  if (col_var) {
52  return codegenColumn(col_var, fetch_columns, co);
53  }
54  auto constant = dynamic_cast<const Analyzer::Constant*>(expr);
55  if (constant) {
56  const auto& ti = constant->get_type_info();
57  if (ti.get_type() == kNULLT) {
58  throw std::runtime_error(
59  "NULL type literals are not currently supported in this context.");
60  }
61  if (constant->get_is_null()) {
62  const auto i8p_ty =
63  llvm::PointerType::get(get_int_type(8, executor_->cgen_state_->context_), 0);
64  if (ti.is_string() && ti.get_compression() == kENCODING_NONE) {
65  std::vector<llvm::Value*> null_target_lvs;
66  llvm::StructType* str_view_ty = createStringViewStructType();
67  auto null_str_view_struct_lv = cgen_state_->ir_builder_.CreateAlloca(str_view_ty);
68  // we do not need to fill the values of the string_view struct representing null
69  // string
70  null_target_lvs.push_back(
71  cgen_state_->ir_builder_.CreateLoad(str_view_ty, null_str_view_struct_lv));
72  null_target_lvs.push_back(llvm::ConstantPointerNull::get(
73  llvm::PointerType::get(llvm::IntegerType::get(cgen_state_->context_, 8), 0)));
74  null_target_lvs.push_back(cgen_state_->llInt((int32_t)0));
75  return null_target_lvs;
76  } else if (ti.is_geometry()) {
77  std::vector<llvm::Value*> ret_lvs;
78  // we classify whether the geo col value is null from the rhs of the left-join
79  // qual i.e., table S in the qual: R left join S on R.v = S.v by checking 1)
80  // nulled chunk_iter and 2) its col buffer w/ the size 0 for instance, we consider
81  // the size of col buffer for point as 16 and for other types we use `array_size`
82  // function to determine it, and they all assume valid geos have their buf size >
83  // 0 thus, by returning all bufs' ptr as nullptr we can return 0 as the length of
84  // the nulled geos' col val, and remaining logic can exploit this to finally
85  // return "null" as the col value of the corresponding row in the resultset
86  switch (ti.get_type()) {
87  // outer join's phi node in the generated code expects the # returned lvs'
88  // is the same as `col_ti.get_physical_coord_cols()`
89  // now we have necessary logic to deal with these nullptr and can conclude
90  // it as nulled geo val, i.e., see ResultSetIteration::991
91  case kPOINT:
92  case kMULTIPOINT:
93  case kLINESTRING:
94  return {llvm::ConstantPointerNull::get(i8p_ty)};
95  case kMULTILINESTRING:
96  case kPOLYGON:
97  return {llvm::ConstantPointerNull::get(i8p_ty),
98  llvm::ConstantPointerNull::get(i8p_ty)};
99  case kMULTIPOLYGON:
100  return {llvm::ConstantPointerNull::get(i8p_ty),
101  llvm::ConstantPointerNull::get(i8p_ty),
102  llvm::ConstantPointerNull::get(i8p_ty)};
103  default:
104  CHECK(false);
105  return {nullptr};
106  }
107  } else if (ti.is_array()) {
108  // similar to above nulled geo case, we can use this `nullptr` to guide
109  // `array_buff` and `array_size` functions representing nulled array value
110  return {llvm::ConstantPointerNull::get(i8p_ty)};
111  }
112  return {ti.is_fp()
113  ? static_cast<llvm::Value*>(executor_->cgen_state_->inlineFpNull(ti))
114  : static_cast<llvm::Value*>(executor_->cgen_state_->inlineIntNull(ti))};
115  }
116  if (ti.get_compression() == kENCODING_DICT) {
117  // The dictionary encoding case should be handled by the parent expression
118  // (cast, for now), here is too late to know the dictionary id if not already set
119  CHECK_NE(ti.getStringDictKey().dict_id, 0);
120  return {codegen(constant, ti.get_compression(), ti.getStringDictKey(), co)};
121  }
122  return {codegen(constant, ti.get_compression(), {}, co)};
123  }
124  auto case_expr = dynamic_cast<const Analyzer::CaseExpr*>(expr);
125  if (case_expr) {
126  return {codegen(case_expr, co)};
127  }
128  auto extract_expr = dynamic_cast<const Analyzer::ExtractExpr*>(expr);
129  if (extract_expr) {
130  return {codegen(extract_expr, co)};
131  }
132  auto dateadd_expr = dynamic_cast<const Analyzer::DateaddExpr*>(expr);
133  if (dateadd_expr) {
134  return {codegen(dateadd_expr, co)};
135  }
136  auto datediff_expr = dynamic_cast<const Analyzer::DatediffExpr*>(expr);
137  if (datediff_expr) {
138  return {codegen(datediff_expr, co)};
139  }
140  auto datetrunc_expr = dynamic_cast<const Analyzer::DatetruncExpr*>(expr);
141  if (datetrunc_expr) {
142  return {codegen(datetrunc_expr, co)};
143  }
144  auto charlength_expr = dynamic_cast<const Analyzer::CharLengthExpr*>(expr);
145  if (charlength_expr) {
146  return {codegen(charlength_expr, co)};
147  }
148  auto keyforstring_expr = dynamic_cast<const Analyzer::KeyForStringExpr*>(expr);
149  if (keyforstring_expr) {
150  return {codegen(keyforstring_expr, co)};
151  }
152  auto sample_ratio_expr = dynamic_cast<const Analyzer::SampleRatioExpr*>(expr);
153  if (sample_ratio_expr) {
154  return {codegen(sample_ratio_expr, co)};
155  }
156  auto string_oper_expr = dynamic_cast<const Analyzer::StringOper*>(expr);
157  if (string_oper_expr) {
158  return {codegen(string_oper_expr, co)};
159  }
160  auto cardinality_expr = dynamic_cast<const Analyzer::CardinalityExpr*>(expr);
161  if (cardinality_expr) {
162  return {codegen(cardinality_expr, co)};
163  }
164  auto like_expr = dynamic_cast<const Analyzer::LikeExpr*>(expr);
165  if (like_expr) {
166  return {codegen(like_expr, co)};
167  }
168  auto regexp_expr = dynamic_cast<const Analyzer::RegexpExpr*>(expr);
169  if (regexp_expr) {
170  return {codegen(regexp_expr, co)};
171  }
172  auto width_bucket_expr = dynamic_cast<const Analyzer::WidthBucketExpr*>(expr);
173  if (width_bucket_expr) {
174  return {codegen(width_bucket_expr, co)};
175  }
176 
177  auto ml_predict_expr = dynamic_cast<const Analyzer::MLPredictExpr*>(expr);
178  if (ml_predict_expr) {
179  return {codegen(ml_predict_expr, co)};
180  }
181 
182  auto pca_project_expr = dynamic_cast<const Analyzer::PCAProjectExpr*>(expr);
183  if (pca_project_expr) {
184  return {codegen(pca_project_expr, co)};
185  }
186 
187  auto likelihood_expr = dynamic_cast<const Analyzer::LikelihoodExpr*>(expr);
188  if (likelihood_expr) {
189  return {codegen(likelihood_expr->get_arg(), fetch_columns, co)};
190  }
191  auto in_expr = dynamic_cast<const Analyzer::InValues*>(expr);
192  if (in_expr) {
193  return {codegen(in_expr, co)};
194  }
195  auto in_integer_set_expr = dynamic_cast<const Analyzer::InIntegerSet*>(expr);
196  if (in_integer_set_expr) {
197  return {codegen(in_integer_set_expr, co)};
198  }
199  auto function_oper_with_custom_type_handling_expr =
200  dynamic_cast<const Analyzer::FunctionOperWithCustomTypeHandling*>(expr);
201  if (function_oper_with_custom_type_handling_expr) {
203  function_oper_with_custom_type_handling_expr, co)};
204  }
205  auto array_oper_expr = dynamic_cast<const Analyzer::ArrayExpr*>(expr);
206  if (array_oper_expr) {
207  return {codegenArrayExpr(array_oper_expr, co)};
208  }
209  auto geo_uop = dynamic_cast<const Analyzer::GeoUOper*>(expr);
210  if (geo_uop) {
211  return {codegenGeoUOper(geo_uop, co)};
212  }
213  auto geo_binop = dynamic_cast<const Analyzer::GeoBinOper*>(expr);
214  if (geo_binop) {
215  return {codegenGeoBinOper(geo_binop, co)};
216  }
217  auto function_oper_expr = dynamic_cast<const Analyzer::FunctionOper*>(expr);
218  if (function_oper_expr) {
219  return {codegenFunctionOper(function_oper_expr, co)};
220  }
221  auto geo_expr = dynamic_cast<const Analyzer::GeoExpr*>(expr);
222  if (geo_expr) {
223  return codegenGeoExpr(geo_expr, co);
224  }
225  if (dynamic_cast<const Analyzer::OffsetInFragment*>(expr)) {
226  return {posArg(nullptr)};
227  }
228  if (dynamic_cast<const Analyzer::WindowFunction*>(expr)) {
229  throw NativeExecutionError("Window expression not supported in this context");
230  }
231  abort();
232 }
233 
234 llvm::Value* CodeGenerator::codegen(const Analyzer::BinOper* bin_oper,
235  const CompilationOptions& co) {
237  const auto optype = bin_oper->get_optype();
238  if (IS_ARITHMETIC(optype)) {
239  return codegenArith(bin_oper, co);
240  }
241  if (IS_COMPARISON(optype)) {
242  return codegenCmp(bin_oper, co);
243  }
244  if (IS_LOGIC(optype)) {
245  return codegenLogical(bin_oper, co);
246  }
247  if (optype == kARRAY_AT) {
248  return codegenArrayAt(bin_oper, co);
249  }
250  abort();
251 }
252 
253 llvm::Value* CodeGenerator::codegen(const Analyzer::UOper* u_oper,
254  const CompilationOptions& co) {
256  const auto optype = u_oper->get_optype();
257  switch (optype) {
258  case kNOT: {
259  return codegenLogical(u_oper, co);
260  }
261  case kCAST: {
262  return codegenCast(u_oper, co);
263  }
264  case kUMINUS: {
265  return codegenUMinus(u_oper, co);
266  }
267  case kISNULL: {
268  return codegenIsNull(u_oper, co);
269  }
270  case kUNNEST:
271  return codegenUnnest(u_oper, co);
272  default:
273  UNREACHABLE();
274  }
275  return nullptr;
276 }
277 
279  const CompilationOptions& co) {
281  auto input_expr = expr->get_arg();
282  CHECK(input_expr);
283 
284  auto double_lv = codegen(input_expr, true, co);
285  CHECK_EQ(size_t(1), double_lv.size());
286 
287  std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
288  const bool is_nullable = !input_expr->get_type_info().get_notnull();
289  if (is_nullable) {
290  nullcheck_codegen = std::make_unique<NullCheckCodegen>(cgen_state_,
291  executor(),
292  double_lv.front(),
293  input_expr->get_type_info(),
294  "sample_ratio_nullcheck");
295  }
296  CHECK_EQ(input_expr->get_type_info().get_type(), kDOUBLE);
297  std::vector<llvm::Value*> args{double_lv[0], posArg(nullptr)};
298  auto ret = cgen_state_->emitCall("sample_ratio", args);
299  if (nullcheck_codegen) {
300  ret = nullcheck_codegen->finalize(ll_bool(false, cgen_state_->context_), ret);
301  }
302  return ret;
303 }
304 
306  const CompilationOptions& co) {
308  auto target_value_expr = expr->get_target_value();
309  auto lower_bound_expr = expr->get_lower_bound();
310  auto upper_bound_expr = expr->get_upper_bound();
311  auto partition_count_expr = expr->get_partition_count();
312  CHECK(target_value_expr);
313  CHECK(lower_bound_expr);
314  CHECK(upper_bound_expr);
315  CHECK(partition_count_expr);
316 
317  auto is_constant_expr = [](const Analyzer::Expr* expr) {
318  auto target_expr = expr;
319  if (auto cast_expr = dynamic_cast<const Analyzer::UOper*>(expr)) {
320  if (cast_expr->get_optype() == SQLOps::kCAST) {
321  target_expr = cast_expr->get_operand();
322  }
323  }
324  // there are more complex constant expr like 1+2, 1/2*3, and so on
325  // but when considering a typical usage of width_bucket function
326  // it is sufficient to consider a singleton constant expr
327  auto constant_expr = dynamic_cast<const Analyzer::Constant*>(target_expr);
328  if (constant_expr) {
329  return true;
330  }
331  return false;
332  };
333  if (is_constant_expr(lower_bound_expr) && is_constant_expr(upper_bound_expr) &&
334  is_constant_expr(partition_count_expr)) {
335  expr->set_constant_expr();
336  }
337  // compute width_bucket's expresn range and check the possibility of avoiding oob check
338  auto col_range =
339  getExpressionRange(expr,
341  executor_,
342  boost::make_optional(plan_state_->getSimpleQuals()));
343  // check whether target_expr is valid
344  if (col_range.getType() == ExpressionRangeType::Integer &&
345  !expr->can_skip_out_of_bound_check() && col_range.getIntMin() > 0 &&
346  col_range.getIntMax() <= expr->get_partition_count_val()) {
347  // check whether target_col is not-nullable or has filter expr on it
348  if (!col_range.hasNulls()) {
349  // Even if the target_expr has its filter expression, target_col_range may exactly
350  // the same with the col_range of the target_expr's operand col,
351  // i.e., SELECT WIDTH_BUCKET(v1, 1, 10, 10) FROM T WHERE v1 != 1;
352  // In that query, col_range of v1 with/without considering the filter expression
353  // v1 != 1 have exactly the same col ranges, so we cannot recognize the existence
354  // of the filter expression based on them. Also, is (not) null is located in
355  // FilterNode, so we cannot trace it in here.
356  // todo (yoonmin): relax this to allow skipping oob check more cases
357  expr->skip_out_of_bound_check();
358  }
359  }
360  return expr->is_constant_expr() ? codegenConstantWidthBucketExpr(expr, co)
361  : codegenWidthBucketExpr(expr, co);
362 }
363 
365  const Analyzer::WidthBucketExpr* expr,
366  const CompilationOptions& co) {
367  auto target_value_expr = expr->get_target_value();
368  auto lower_bound_expr = expr->get_lower_bound();
369  auto upper_bound_expr = expr->get_upper_bound();
370  auto partition_count_expr = expr->get_partition_count();
371 
372  auto num_partitions = expr->get_partition_count_val();
373  if (num_partitions < 1 || num_partitions > INT32_MAX) {
374  throw std::runtime_error(
375  "PARTITION_COUNT expression of width_bucket function should be in a valid "
376  "range: 0 < PARTITION_COUNT <= 2147483647");
377  }
378  double lower = expr->get_bound_val(lower_bound_expr);
379  double upper = expr->get_bound_val(upper_bound_expr);
380  if (lower == upper) {
381  throw std::runtime_error(
382  "LOWER_BOUND and UPPER_BOUND expressions of width_bucket function cannot have "
383  "the same constant value");
384  }
385  if (lower == NULL_DOUBLE || upper == NULL_DOUBLE) {
386  throw std::runtime_error(
387  "Both LOWER_BOUND and UPPER_BOUND of width_bucket function should be finite "
388  "numeric constants.");
389  }
390 
391  bool const reversed = lower > upper;
392  double const scale_factor = num_partitions / (reversed ? lower - upper : upper - lower);
393  std::string func_name = reversed ? "width_bucket_reversed" : "width_bucket";
394 
395  auto get_double_constant_lvs = [this, &co](double const_val) {
396  Datum d;
397  d.doubleval = const_val;
398  auto double_const_expr =
399  makeExpr<Analyzer::Constant>(SQLTypeInfo(kDOUBLE, false), false, d);
400  return codegen(double_const_expr.get(), false, co);
401  };
402 
403  auto target_value_ti = target_value_expr->get_type_info();
404  auto target_value_expr_lvs = codegen(target_value_expr, true, co);
405  CHECK_EQ(size_t(1), target_value_expr_lvs.size());
406  auto lower_expr_lvs = codegen(lower_bound_expr, true, co);
407  CHECK_EQ(size_t(1), lower_expr_lvs.size());
408  auto scale_factor_lvs = get_double_constant_lvs(scale_factor);
409  CHECK_EQ(size_t(1), scale_factor_lvs.size());
410 
411  std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
412  lower_expr_lvs[0]};
413  if (expr->can_skip_out_of_bound_check()) {
414  func_name += "_no_oob_check";
415  width_bucket_args.push_back(scale_factor_lvs[0]);
416  } else {
417  auto upper_expr_lvs = codegen(upper_bound_expr, true, co);
418  CHECK_EQ(size_t(1), upper_expr_lvs.size());
419  auto partition_count_expr_lvs = codegen(partition_count_expr, true, co);
420  CHECK_EQ(size_t(1), partition_count_expr_lvs.size());
421  width_bucket_args.push_back(upper_expr_lvs[0]);
422  width_bucket_args.push_back(scale_factor_lvs[0]);
423  width_bucket_args.push_back(partition_count_expr_lvs[0]);
424  if (!target_value_ti.get_notnull()) {
425  func_name += "_nullable";
426  auto translated_null_value = target_value_ti.is_fp()
427  ? inline_fp_null_val(target_value_ti)
428  : inline_int_null_val(target_value_ti);
429  auto null_value_lvs = get_double_constant_lvs(translated_null_value);
430  CHECK_EQ(size_t(1), null_value_lvs.size());
431  width_bucket_args.push_back(null_value_lvs[0]);
432  }
433  }
434  return cgen_state_->emitCall(func_name, width_bucket_args);
435 }
436 
438  const CompilationOptions& co) {
439  auto target_value_expr = expr->get_target_value();
440  auto lower_bound_expr = expr->get_lower_bound();
441  auto upper_bound_expr = expr->get_upper_bound();
442  auto partition_count_expr = expr->get_partition_count();
443 
444  std::string func_name = "width_bucket_expr";
445  bool nullable_expr = false;
446  if (expr->can_skip_out_of_bound_check()) {
447  func_name += "_no_oob_check";
448  } else if (!target_value_expr->get_type_info().get_notnull()) {
449  func_name += "_nullable";
450  nullable_expr = true;
451  }
452 
453  auto target_value_expr_lvs = codegen(target_value_expr, true, co);
454  CHECK_EQ(size_t(1), target_value_expr_lvs.size());
455  auto lower_bound_expr_lvs = codegen(lower_bound_expr, true, co);
456  CHECK_EQ(size_t(1), lower_bound_expr_lvs.size());
457  auto upper_bound_expr_lvs = codegen(upper_bound_expr, true, co);
458  CHECK_EQ(size_t(1), upper_bound_expr_lvs.size());
459  auto partition_count_expr_lvs = codegen(partition_count_expr, true, co);
460  CHECK_EQ(size_t(1), partition_count_expr_lvs.size());
461  auto target_value_ti = target_value_expr->get_type_info();
462  auto null_value_lv = cgen_state_->inlineFpNull(target_value_ti);
463 
464  // check partition count : 1 ~ INT32_MAX
465  // INT32_MAX will be checked during casting by OVERFLOW checking step
466  auto partition_count_ti = partition_count_expr->get_type_info();
467  CHECK(partition_count_ti.is_integer());
468  auto int32_ti = SQLTypeInfo(kINT, partition_count_ti.get_notnull());
469  auto partition_count_expr_lv =
470  codegenCastBetweenIntTypes(partition_count_expr_lvs[0],
471  partition_count_ti,
472  int32_ti,
473  partition_count_ti.get_size() < int32_ti.get_size());
474  llvm::Value* chosen_min = cgen_state_->llInt(static_cast<int32_t>(0));
475  llvm::Value* partition_count_min =
476  cgen_state_->ir_builder_.CreateICmpSLE(partition_count_expr_lv, chosen_min);
477  llvm::BasicBlock* width_bucket_partition_count_ok_bb =
478  llvm::BasicBlock::Create(cgen_state_->context_,
479  "width_bucket_partition_count_ok_bb",
481  llvm::BasicBlock* width_bucket_argument_check_fail_bb =
482  llvm::BasicBlock::Create(cgen_state_->context_,
483  "width_bucket_argument_check_fail_bb",
485  cgen_state_->ir_builder_.CreateCondBr(partition_count_min,
486  width_bucket_argument_check_fail_bb,
487  width_bucket_partition_count_ok_bb);
488  cgen_state_->ir_builder_.SetInsertPoint(width_bucket_argument_check_fail_bb);
489  cgen_state_->ir_builder_.CreateRet(
491  cgen_state_->ir_builder_.SetInsertPoint(width_bucket_partition_count_ok_bb);
492 
493  llvm::BasicBlock* width_bucket_bound_check_ok_bb =
494  llvm::BasicBlock::Create(cgen_state_->context_,
495  "width_bucket_bound_check_ok_bb",
497  llvm::Value* bound_check{nullptr};
498  if (lower_bound_expr->get_type_info().get_notnull() &&
499  upper_bound_expr->get_type_info().get_notnull()) {
500  bound_check = cgen_state_->ir_builder_.CreateFCmpOEQ(
501  lower_bound_expr_lvs[0], upper_bound_expr_lvs[0], "bound_check");
502  } else {
503  std::vector<llvm::Value*> bound_check_args{
504  lower_bound_expr_lvs[0],
505  upper_bound_expr_lvs[0],
506  null_value_lv,
507  cgen_state_->llInt(static_cast<int8_t>(1))};
508  bound_check = toBool(cgen_state_->emitCall("eq_double_nullable", bound_check_args));
509  }
510  cgen_state_->ir_builder_.CreateCondBr(
511  bound_check, width_bucket_argument_check_fail_bb, width_bucket_bound_check_ok_bb);
512  cgen_state_->ir_builder_.SetInsertPoint(width_bucket_bound_check_ok_bb);
514  auto reversed_expr = toBool(codegenCmp(SQLOps::kGT,
515  kONE,
516  lower_bound_expr_lvs,
517  lower_bound_expr->get_type_info(),
518  upper_bound_expr,
519  co));
520  auto lower_bound_expr_lv = lower_bound_expr_lvs[0];
521  auto upper_bound_expr_lv = upper_bound_expr_lvs[0];
522  std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
523  reversed_expr,
524  lower_bound_expr_lv,
525  upper_bound_expr_lv,
526  partition_count_expr_lv};
527  if (nullable_expr) {
528  width_bucket_args.push_back(null_value_lv);
529  }
530  return cgen_state_->emitCall(func_name, width_bucket_args);
531 }
532 
533 namespace {
534 
536  const std::shared_ptr<Analyzer::Expr>& qual) {
537  const auto qual_cf = qual_to_conjunctive_form(qual);
538  ra_exe_unit.simple_quals.insert(ra_exe_unit.simple_quals.end(),
539  qual_cf.simple_quals.begin(),
540  qual_cf.simple_quals.end());
541  ra_exe_unit.quals.insert(
542  ra_exe_unit.quals.end(), qual_cf.quals.begin(), qual_cf.quals.end());
543 }
544 
546  const ExecutionOptions& eo,
547  const std::vector<InputTableInfo>& query_infos,
548  const size_t level_idx,
549  const std::string& fail_reason) {
550  if (ra_exe_unit.input_descs.size() < 2) {
551  // We only support loop join at the end of folded joins
552  // where ra_exe_unit.input_descs.size() > 2 for now.
553  throw std::runtime_error("Hash join failed, reason(s): " + fail_reason +
554  " | Incorrect # tables for executing loop join");
555  }
556  const auto loop_join_size = get_loop_join_size(query_infos, ra_exe_unit);
557  const bool has_loop_size_hint =
559  const size_t loop_join_size_threshold =
560  has_loop_size_hint ? ra_exe_unit.query_hint.loop_join_inner_table_max_num_rows
562  if (eo.allow_loop_joins) {
563  if (has_loop_size_hint && loop_join_size_threshold < loop_join_size) {
564  throw std::runtime_error(
565  "Hash join failed, reason(s): " + fail_reason +
566  " | Cannot fall back to loop join for non-trivial inner table size");
567  }
568  return;
569  }
570  if (level_idx + 1 != ra_exe_unit.join_quals.size()) {
571  throw std::runtime_error(
572  "Hash join failed, reason(s): " + fail_reason +
573  " | Cannot fall back to loop join for intermediate join quals");
574  }
575  if (loop_join_size_threshold < loop_join_size) {
576  throw std::runtime_error(
577  "Hash join failed, reason(s): " + fail_reason +
578  " | Cannot fall back to loop join for non-trivial inner table size");
579  }
580  if (ra_exe_unit.query_hint.isHintRegistered(kDisableLoopJoin)) {
581  throw std::runtime_error("Hash join failed, reason(s): " + fail_reason +
582  " | Loop join is disabled by user");
583  }
584 }
585 
586 void check_valid_join_qual(std::shared_ptr<Analyzer::BinOper>& bin_oper) {
587  // check whether a join qual is valid before entering the hashtable build and codegen
588 
589  auto lhs_cv = dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_left_operand());
590  auto rhs_cv = dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_right_operand());
591  if (lhs_cv && rhs_cv && !bin_oper->is_bbox_intersect_oper()) {
592  auto lhs_type = lhs_cv->get_type_info().get_type();
593  auto rhs_type = rhs_cv->get_type_info().get_type();
594  // check #1. avoid a join btw full array columns
595  if (lhs_type == SQLTypes::kARRAY && rhs_type == SQLTypes::kARRAY) {
596  throw std::runtime_error(
597  "Join operation between full array columns (i.e., R.arr = S.arr) instead of "
598  "indexed array columns (i.e., R.arr[1] = S.arr[2]) is not supported yet.");
599  }
600  }
601 }
602 
603 } // namespace
604 
605 std::vector<JoinLoop> Executor::buildJoinLoops(
606  RelAlgExecutionUnit& ra_exe_unit,
607  const CompilationOptions& co,
608  const ExecutionOptions& eo,
609  const std::vector<InputTableInfo>& query_infos,
610  ColumnCacheMap& column_cache) {
611  INJECT_TIMER(buildJoinLoops);
613  std::vector<JoinLoop> join_loops;
614  for (size_t level_idx = 0, current_hash_table_idx = 0;
615  level_idx < ra_exe_unit.join_quals.size();
616  ++level_idx) {
617  const auto& current_level_join_conditions = ra_exe_unit.join_quals[level_idx];
618  std::vector<std::string> fail_reasons;
619  const auto current_level_hash_table =
620  buildCurrentLevelHashTable(current_level_join_conditions,
621  level_idx,
622  ra_exe_unit,
623  co,
624  query_infos,
625  column_cache,
626  fail_reasons);
627  const auto found_outer_join_matches_cb =
628  [this, level_idx](llvm::Value* found_outer_join_matches) {
632  found_outer_join_matches;
633  };
634  const auto is_deleted_cb = buildIsDeletedCb(ra_exe_unit, level_idx, co);
635  auto rem_left_join_quals_it =
637  bool has_remaining_left_join_quals =
638  rem_left_join_quals_it != plan_state_->left_join_non_hashtable_quals_.end() &&
639  !rem_left_join_quals_it->second.empty();
640  const auto outer_join_condition_remaining_quals_cb =
641  [this, level_idx, &co](const std::vector<llvm::Value*>& prev_iters) {
642  // when we have multiple quals for the left join in the current join level
643  // we first try to build a hashtable by using one of the possible qual,
644  // and deal with remaining quals as extra join conditions
645  FetchCacheAnchor anchor(cgen_state_.get());
646  addJoinLoopIterator(prev_iters, level_idx + 1);
647  llvm::Value* left_join_cond = cgen_state_->llBool(true);
648  CodeGenerator code_generator(this);
649  auto it = plan_state_->left_join_non_hashtable_quals_.find(level_idx);
650  if (it != plan_state_->left_join_non_hashtable_quals_.end()) {
651  for (auto expr : it->second) {
652  left_join_cond = cgen_state_->ir_builder_.CreateAnd(
653  left_join_cond,
654  code_generator.toBool(
655  code_generator.codegen(expr.get(), true, co).front()));
656  }
657  }
658  return left_join_cond;
659  };
660  if (current_level_hash_table) {
661  const auto hoisted_filters_cb = buildHoistLeftHandSideFiltersCb(
662  ra_exe_unit, level_idx, current_level_hash_table->getInnerTableId(), co);
663  if (current_level_hash_table->getHashType() == HashType::OneToOne) {
664  join_loops.emplace_back(
665  /*kind=*/JoinLoopKind::Singleton,
666  /*type=*/current_level_join_conditions.type,
667  /*iteration_domain_codegen=*/
668  [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
669  const std::vector<llvm::Value*>& prev_iters) {
670  addJoinLoopIterator(prev_iters, level_idx);
671  JoinLoopDomain domain{{0}};
672  domain.slot_lookup_result =
673  current_level_hash_table->codegenSlot(co, current_hash_table_idx);
674  return domain;
675  },
676  /*outer_condition_match=*/
677  current_level_join_conditions.type == JoinType::LEFT &&
678  has_remaining_left_join_quals
679  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
680  outer_join_condition_remaining_quals_cb)
681  : nullptr,
682  /*found_outer_matches=*/current_level_join_conditions.type == JoinType::LEFT
683  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
684  : nullptr,
685  /*hoisted_filters=*/hoisted_filters_cb,
686  /*is_deleted=*/is_deleted_cb,
687  /*nested_loop_join=*/false);
688  } else if (auto range_join_table =
689  dynamic_cast<RangeJoinHashTable*>(current_level_hash_table.get())) {
690  join_loops.emplace_back(
691  /* kind= */ JoinLoopKind::MultiSet,
692  /* type= */ current_level_join_conditions.type,
693  /* iteration_domain_codegen= */
694  [this,
695  range_join_table,
696  current_hash_table_idx,
697  level_idx,
698  current_level_hash_table,
699  &co](const std::vector<llvm::Value*>& prev_iters) {
700  addJoinLoopIterator(prev_iters, level_idx);
701  JoinLoopDomain domain{{0}};
702  CHECK(!prev_iters.empty());
703  const auto matching_set = range_join_table->codegenMatchingSetWithOffset(
704  co, current_hash_table_idx, prev_iters.back());
705  domain.values_buffer = matching_set.elements;
706  domain.element_count = matching_set.count;
707  return domain;
708  },
709  /* outer_condition_match= */
710  current_level_join_conditions.type == JoinType::LEFT
711  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
712  outer_join_condition_remaining_quals_cb)
713  : nullptr,
714  /* found_outer_matches= */
715  current_level_join_conditions.type == JoinType::LEFT
716  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
717  : nullptr,
718  /* hoisted_filters= */ nullptr, // <<! TODO
719  /* is_deleted= */ is_deleted_cb,
720  /*nested_loop_join=*/false);
721  } else {
722  join_loops.emplace_back(
723  /*kind=*/JoinLoopKind::Set,
724  /*type=*/current_level_join_conditions.type,
725  /*iteration_domain_codegen=*/
726  [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
727  const std::vector<llvm::Value*>& prev_iters) {
728  addJoinLoopIterator(prev_iters, level_idx);
729  JoinLoopDomain domain{{0}};
730  const auto matching_set = current_level_hash_table->codegenMatchingSet(
731  co, current_hash_table_idx);
732  domain.values_buffer = matching_set.elements;
733  domain.element_count = matching_set.count;
734  return domain;
735  },
736  /*outer_condition_match=*/
737  current_level_join_conditions.type == JoinType::LEFT
738  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
739  outer_join_condition_remaining_quals_cb)
740  : nullptr,
741  /*found_outer_matches=*/current_level_join_conditions.type == JoinType::LEFT
742  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
743  : nullptr,
744  /*hoisted_filters=*/hoisted_filters_cb,
745  /*is_deleted=*/is_deleted_cb,
746  /*nested_loop_join=*/false);
747  }
748  ++current_hash_table_idx;
749  } else {
750  const auto fail_reasons_str = current_level_join_conditions.quals.empty()
751  ? "No equijoin expression found"
752  : boost::algorithm::join(fail_reasons, " | ");
754  ra_exe_unit, eo, query_infos, level_idx, fail_reasons_str);
755  // Callback provided to the `JoinLoop` framework to evaluate the (outer) join
756  // condition.
757  VLOG(1) << "Unable to build hash table, falling back to loop join: "
758  << fail_reasons_str;
759  const auto outer_join_condition_cb =
760  [this, level_idx, &co, &current_level_join_conditions](
761  const std::vector<llvm::Value*>& prev_iters) {
762  // The values generated for the match path don't dominate all uses
763  // since on the non-match path nulls are generated. Reset the cache
764  // once the condition is generated to avoid incorrect reuse.
765  FetchCacheAnchor anchor(cgen_state_.get());
766  addJoinLoopIterator(prev_iters, level_idx + 1);
767  llvm::Value* left_join_cond = cgen_state_->llBool(true);
768  CodeGenerator code_generator(this);
769  for (auto expr : current_level_join_conditions.quals) {
770  left_join_cond = cgen_state_->ir_builder_.CreateAnd(
771  left_join_cond,
772  code_generator.toBool(
773  code_generator.codegen(expr.get(), true, co).front()));
774  }
775  return left_join_cond;
776  };
777  join_loops.emplace_back(
778  /*kind=*/JoinLoopKind::UpperBound,
779  /*type=*/current_level_join_conditions.type,
780  /*iteration_domain_codegen=*/
781  [this, level_idx](const std::vector<llvm::Value*>& prev_iters) {
782  addJoinLoopIterator(prev_iters, level_idx);
783  JoinLoopDomain domain{{0}};
784  auto* arg = get_arg_by_name(cgen_state_->row_func_, "num_rows_per_scan");
785  const auto rows_per_scan_ptr = cgen_state_->ir_builder_.CreateGEP(
786  arg->getType()->getScalarType()->getPointerElementType(),
787  arg,
788  cgen_state_->llInt(int32_t(level_idx + 1)));
789  domain.upper_bound = cgen_state_->ir_builder_.CreateLoad(
790  rows_per_scan_ptr->getType()->getPointerElementType(),
791  rows_per_scan_ptr,
792  "num_rows_per_scan");
793  return domain;
794  },
795  /*outer_condition_match=*/
796  current_level_join_conditions.type == JoinType::LEFT
797  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
798  outer_join_condition_cb)
799  : nullptr,
800  /*found_outer_matches=*/
801  current_level_join_conditions.type == JoinType::LEFT
802  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
803  : nullptr,
804  /*hoisted_filters=*/nullptr,
805  /*is_deleted=*/is_deleted_cb,
806  /*nested_loop_join=*/true);
807  }
808  }
809  return join_loops;
810 }
811 
812 namespace {
813 
814 class ExprTableIdVisitor : public ScalarExprVisitor<std::set<shared::TableKey>> {
815  protected:
816  std::set<shared::TableKey> visitColumnVar(
817  const Analyzer::ColumnVar* col_expr) const final {
818  return {col_expr->getTableKey()};
819  }
820 
821  std::set<shared::TableKey> visitFunctionOper(
822  const Analyzer::FunctionOper* func_expr) const final {
823  std::set<shared::TableKey> ret;
824  for (size_t i = 0; i < func_expr->getArity(); i++) {
825  ret = aggregateResult(ret, visit(func_expr->getArg(i)));
826  }
827  return ret;
828  }
829 
830  std::set<shared::TableKey> visitBinOper(const Analyzer::BinOper* bin_oper) const final {
831  std::set<shared::TableKey> ret;
832  ret = aggregateResult(ret, visit(bin_oper->get_left_operand()));
833  return aggregateResult(ret, visit(bin_oper->get_right_operand()));
834  }
835 
836  std::set<shared::TableKey> visitUOper(const Analyzer::UOper* u_oper) const final {
837  return visit(u_oper->get_operand());
838  }
839 
840  std::set<shared::TableKey> aggregateResult(
841  const std::set<shared::TableKey>& aggregate,
842  const std::set<shared::TableKey>& next_result) const final {
843  auto ret = aggregate; // copy
844  for (const auto& el : next_result) {
845  ret.insert(el);
846  }
847  return ret;
848  }
849 };
850 
851 } // namespace
852 
854  const RelAlgExecutionUnit& ra_exe_unit,
855  const size_t level_idx,
856  const shared::TableKey& inner_table_id,
857  const CompilationOptions& co) {
859  return nullptr;
860  }
861 
862  const auto& current_level_join_conditions = ra_exe_unit.join_quals[level_idx];
863  if (level_idx == 0 && current_level_join_conditions.type == JoinType::LEFT) {
864  const auto& condition = current_level_join_conditions.quals.front();
865  const auto bin_oper = dynamic_cast<const Analyzer::BinOper*>(condition.get());
866  CHECK(bin_oper) << condition->toString();
867  const auto rhs =
868  dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_right_operand());
869  const auto lhs =
870  dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_left_operand());
871  if (lhs && rhs && lhs->getTableKey() != rhs->getTableKey()) {
872  const Analyzer::ColumnVar* selected_lhs{nullptr};
873  // grab the left hand side column -- this is somewhat similar to normalize column
874  // pair, and a better solution may be to hoist that function out of the join
875  // framework and normalize columns at the top of build join loops
876  if (lhs->getTableKey() == inner_table_id) {
877  selected_lhs = rhs;
878  } else if (rhs->getTableKey() == inner_table_id) {
879  selected_lhs = lhs;
880  }
881  if (selected_lhs) {
882  std::list<std::shared_ptr<Analyzer::Expr>> hoisted_quals;
883  // get all LHS-only filters
884  auto should_hoist_qual = [&hoisted_quals](const auto& qual,
885  const shared::TableKey& table_key) {
886  CHECK(qual);
887 
888  ExprTableIdVisitor visitor;
889  const auto table_keys = visitor.visit(qual.get());
890  if (table_keys.size() == 1 && table_keys.find(table_key) != table_keys.end()) {
891  hoisted_quals.push_back(qual);
892  }
893  };
894  for (const auto& qual : ra_exe_unit.simple_quals) {
895  should_hoist_qual(qual, selected_lhs->getTableKey());
896  }
897  for (const auto& qual : ra_exe_unit.quals) {
898  should_hoist_qual(qual, selected_lhs->getTableKey());
899  }
900 
901  // build the filters callback and return it
902  if (!hoisted_quals.empty()) {
903  return [this, hoisted_quals, co](llvm::BasicBlock* true_bb,
904  llvm::BasicBlock* exit_bb,
905  const std::string& loop_name,
906  llvm::Function* parent_func,
907  CgenState* cgen_state) -> llvm::BasicBlock* {
908  // make sure we have quals to hoist
909  bool has_quals_to_hoist = false;
910  for (const auto& qual : hoisted_quals) {
911  // check to see if the filter was previously hoisted. if all filters were
912  // previously hoisted, this callback becomes a noop
913  if (plan_state_->hoisted_filters_.count(qual) == 0) {
914  has_quals_to_hoist = true;
915  break;
916  }
917  }
918 
919  if (!has_quals_to_hoist) {
920  return nullptr;
921  }
922 
923  AUTOMATIC_IR_METADATA(cgen_state);
924 
925  llvm::IRBuilder<>& builder = cgen_state->ir_builder_;
926  auto& context = builder.getContext();
927 
928  const auto filter_bb =
929  llvm::BasicBlock::Create(context,
930  "hoisted_left_join_filters_" + loop_name,
931  parent_func,
932  /*insert_before=*/true_bb);
933  builder.SetInsertPoint(filter_bb);
934 
935  llvm::Value* filter_lv = cgen_state_->llBool(true);
936  CodeGenerator code_generator(this);
937  CHECK(plan_state_);
938  for (const auto& qual : hoisted_quals) {
939  if (plan_state_->hoisted_filters_.insert(qual).second) {
940  // qual was inserted into the hoisted filters map, which means we have not
941  // seen this qual before. Generate filter.
942  VLOG(1) << "Generating code for hoisted left hand side qualifier "
943  << qual->toString();
944  auto cond = code_generator.toBool(
945  code_generator.codegen(qual.get(), true, co).front());
946  filter_lv = builder.CreateAnd(filter_lv, cond);
947  }
948  }
949  CHECK(filter_lv->getType()->isIntegerTy(1));
950 
951  builder.CreateCondBr(filter_lv, true_bb, exit_bb);
952  return filter_bb;
953  };
954  }
955  }
956  }
957  }
958  return nullptr;
959 }
960 
961 std::function<llvm::Value*(const std::vector<llvm::Value*>&, llvm::Value*)>
963  const size_t level_idx,
964  const CompilationOptions& co) {
965  AUTOMATIC_IR_METADATA(cgen_state_.get());
966  if (!co.filter_on_deleted_column) {
967  return nullptr;
968  }
969  CHECK_LT(level_idx + 1, ra_exe_unit.input_descs.size());
970  const auto input_desc = ra_exe_unit.input_descs[level_idx + 1];
971  if (input_desc.getSourceType() != InputSourceType::TABLE) {
972  return nullptr;
973  }
974 
975  const auto deleted_cd = plan_state_->getDeletedColForTable(input_desc.getTableKey());
976  if (!deleted_cd) {
977  return nullptr;
978  }
979  CHECK(deleted_cd->columnType.is_boolean());
980  const auto deleted_expr = makeExpr<Analyzer::ColumnVar>(
981  deleted_cd->columnType,
982  shared::ColumnKey{input_desc.getTableKey(), deleted_cd->columnId},
983  input_desc.getNestLevel());
984  return [this, deleted_expr, level_idx, &co](const std::vector<llvm::Value*>& prev_iters,
985  llvm::Value* have_more_inner_rows) {
986  const auto matching_row_index = addJoinLoopIterator(prev_iters, level_idx + 1);
987  // Avoid fetching the deleted column from a position which is not valid.
988  // An invalid position can be returned by a one to one hash lookup (negative)
989  // or at the end of iteration over a set of matching values.
990  llvm::Value* is_valid_it{nullptr};
991  if (have_more_inner_rows) {
992  is_valid_it = have_more_inner_rows;
993  } else {
994  is_valid_it = cgen_state_->ir_builder_.CreateICmp(
995  llvm::ICmpInst::ICMP_SGE, matching_row_index, cgen_state_->llInt<int64_t>(0));
996  }
997  const auto it_valid_bb = llvm::BasicBlock::Create(
998  cgen_state_->context_, "it_valid", cgen_state_->current_func_);
999  const auto it_not_valid_bb = llvm::BasicBlock::Create(
1000  cgen_state_->context_, "it_not_valid", cgen_state_->current_func_);
1001  cgen_state_->ir_builder_.CreateCondBr(is_valid_it, it_valid_bb, it_not_valid_bb);
1002  const auto row_is_deleted_bb = llvm::BasicBlock::Create(
1003  cgen_state_->context_, "row_is_deleted", cgen_state_->current_func_);
1004  cgen_state_->ir_builder_.SetInsertPoint(it_valid_bb);
1005  CodeGenerator code_generator(this);
1006  const auto row_is_deleted = code_generator.toBool(
1007  code_generator.codegen(deleted_expr.get(), true, co).front());
1008  cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
1009  cgen_state_->ir_builder_.SetInsertPoint(it_not_valid_bb);
1010  const auto row_is_deleted_default = cgen_state_->llBool(false);
1011  cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
1012  cgen_state_->ir_builder_.SetInsertPoint(row_is_deleted_bb);
1013  auto row_is_deleted_or_default =
1014  cgen_state_->ir_builder_.CreatePHI(row_is_deleted->getType(), 2);
1015  row_is_deleted_or_default->addIncoming(row_is_deleted, it_valid_bb);
1016  row_is_deleted_or_default->addIncoming(row_is_deleted_default, it_not_valid_bb);
1017  return row_is_deleted_or_default;
1018  };
1019 }
1020 
1021 std::shared_ptr<HashJoin> Executor::buildCurrentLevelHashTable(
1022  const JoinCondition& current_level_join_conditions,
1023  size_t level_idx,
1024  RelAlgExecutionUnit& ra_exe_unit,
1025  const CompilationOptions& co,
1026  const std::vector<InputTableInfo>& query_infos,
1027  ColumnCacheMap& column_cache,
1028  std::vector<std::string>& fail_reasons) {
1029  AUTOMATIC_IR_METADATA(cgen_state_.get());
1030  std::shared_ptr<HashJoin> current_level_hash_table;
1031  auto handleNonHashtableQual = [&ra_exe_unit, &level_idx, this](
1032  JoinType join_type,
1033  std::shared_ptr<Analyzer::Expr> qual) {
1034  if (join_type == JoinType::LEFT) {
1035  plan_state_->addNonHashtableQualForLeftJoin(level_idx, qual);
1036  } else {
1037  add_qualifier_to_execution_unit(ra_exe_unit, qual);
1038  }
1039  };
1040  for (const auto& join_qual : current_level_join_conditions.quals) {
1041  auto qual_bin_oper = std::dynamic_pointer_cast<Analyzer::BinOper>(join_qual);
1042  if (current_level_hash_table || !qual_bin_oper ||
1043  !IS_EQUIVALENCE(qual_bin_oper->get_optype())) {
1044  handleNonHashtableQual(current_level_join_conditions.type, join_qual);
1045  if (!current_level_hash_table) {
1046  fail_reasons.emplace_back("No equijoin expression found");
1047  }
1048  continue;
1049  }
1050  check_valid_join_qual(qual_bin_oper);
1051  JoinHashTableOrError hash_table_or_error;
1052  if (!current_level_hash_table) {
1053  hash_table_or_error = buildHashTableForQualifier(
1054  qual_bin_oper,
1055  query_infos,
1058  current_level_join_conditions.type,
1060  column_cache,
1061  ra_exe_unit.hash_table_build_plan_dag,
1062  ra_exe_unit.query_hint,
1063  ra_exe_unit.table_id_to_node_map);
1064  current_level_hash_table = hash_table_or_error.hash_table;
1065  }
1066  if (hash_table_or_error.hash_table) {
1067  plan_state_->join_info_.join_hash_tables_.push_back(hash_table_or_error.hash_table);
1068  plan_state_->join_info_.equi_join_tautologies_.push_back(qual_bin_oper);
1069  } else {
1070  fail_reasons.push_back(hash_table_or_error.fail_reason);
1071  if (!current_level_hash_table) {
1072  VLOG(2) << "Building a hashtable based on a qual " << qual_bin_oper->toString()
1073  << " fails: " << hash_table_or_error.fail_reason;
1074  }
1075  handleNonHashtableQual(current_level_join_conditions.type, qual_bin_oper);
1076  }
1077  }
1078  return current_level_hash_table;
1079 }
1080 
1082  if (!cgen_state_->filter_func_) {
1083  return;
1084  }
1085 
1086  // Loop over all the instructions used in the filter func.
1087  // The filter func instructions were generated as if for row func.
1088  // Remap any values used by those instructions to filter func args
1089  // and remember to forward them through the call in the row func.
1090  for (auto bb_it = cgen_state_->filter_func_->begin();
1091  bb_it != cgen_state_->filter_func_->end();
1092  ++bb_it) {
1093  for (auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1094  size_t i = 0;
1095  for (auto op_it = instr_it->value_op_begin(); op_it != instr_it->value_op_end();
1096  ++op_it, ++i) {
1097  llvm::Value* v = *op_it;
1098 
1099  // The last LLVM operand on a call instruction is the function to be called. Never
1100  // remap it.
1101  if (llvm::dyn_cast<const llvm::CallInst>(instr_it) &&
1102  op_it == instr_it->value_op_end() - 1) {
1103  continue;
1104  }
1105 
1106  CHECK(v);
1107  if (auto* instr = llvm::dyn_cast<llvm::Instruction>(v);
1108  instr && instr->getParent() &&
1109  instr->getParent()->getParent() == cgen_state_->row_func_) {
1110  // Remember that this filter func arg is needed.
1111  cgen_state_->filter_func_args_[v] = nullptr;
1112  } else if (auto* argum = llvm::dyn_cast<llvm::Argument>(v);
1113  argum && argum->getParent() == cgen_state_->row_func_) {
1114  // Remember that this filter func arg is needed.
1115  cgen_state_->filter_func_args_[v] = nullptr;
1116  }
1117  }
1118  }
1119  }
1120 
1121  // Create filter_func2 with parameters only for those row func values that are known to
1122  // be used in the filter func code.
1123  std::vector<llvm::Type*> filter_func_arg_types;
1124  filter_func_arg_types.reserve(cgen_state_->filter_func_args_.v_.size());
1125  for (auto& arg : cgen_state_->filter_func_args_.v_) {
1126  filter_func_arg_types.push_back(arg->getType());
1127  }
1128  auto ft = llvm::FunctionType::get(
1129  get_int_type(32, cgen_state_->context_), filter_func_arg_types, false);
1130  cgen_state_->filter_func_->setName("old_filter_func");
1131  auto filter_func2 = llvm::Function::Create(ft,
1132  llvm::Function::ExternalLinkage,
1133  "filter_func",
1134  cgen_state_->filter_func_->getParent());
1135  CHECK_EQ(filter_func2->arg_size(), cgen_state_->filter_func_args_.v_.size());
1136  auto arg_it = cgen_state_->filter_func_args_.begin();
1137  size_t i = 0;
1138  for (llvm::Function::arg_iterator I = filter_func2->arg_begin(),
1139  E = filter_func2->arg_end();
1140  I != E;
1141  ++I, ++arg_it) {
1142  arg_it->second = &*I;
1143  if (arg_it->first->hasName()) {
1144  I->setName(arg_it->first->getName());
1145  } else {
1146  I->setName("extra" + std::to_string(i++));
1147  }
1148  }
1149 
1150  // copy the filter_func function body over
1151  // see
1152  // https://stackoverflow.com/questions/12864106/move-function-body-avoiding-full-cloning/18751365
1153  filter_func2->getBasicBlockList().splice(
1154  filter_func2->begin(), cgen_state_->filter_func_->getBasicBlockList());
1155 
1156  if (cgen_state_->current_func_ == cgen_state_->filter_func_) {
1157  cgen_state_->current_func_ = filter_func2;
1158  }
1159  cgen_state_->filter_func_ = filter_func2;
1160 
1161  // loop over all the operands in the filter func
1162  for (auto bb_it = cgen_state_->filter_func_->begin();
1163  bb_it != cgen_state_->filter_func_->end();
1164  ++bb_it) {
1165  for (auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1166  size_t i = 0;
1167  for (auto op_it = instr_it->op_begin(); op_it != instr_it->op_end(); ++op_it, ++i) {
1168  llvm::Value* v = op_it->get();
1169  if (auto arg_it = cgen_state_->filter_func_args_.find(v);
1170  arg_it != cgen_state_->filter_func_args_.end()) {
1171  // replace row func value with a filter func arg
1172  llvm::Use* use = &*op_it;
1173  use->set(arg_it->second);
1174  }
1175  }
1176  }
1177  }
1178 }
1179 
1180 llvm::Value* Executor::addJoinLoopIterator(const std::vector<llvm::Value*>& prev_iters,
1181  const size_t level_idx) {
1182  AUTOMATIC_IR_METADATA(cgen_state_.get());
1183  // Iterators are added for loop-outer joins when the head of the loop is generated,
1184  // then once again when the body if generated. Allow this instead of special handling
1185  // of call sites.
1186  const auto it = cgen_state_->scan_idx_to_hash_pos_.find(level_idx);
1187  if (it != cgen_state_->scan_idx_to_hash_pos_.end()) {
1188  return it->second;
1189  }
1190  CHECK(!prev_iters.empty());
1191  llvm::Value* matching_row_index = prev_iters.back();
1192  const auto it_ok =
1193  cgen_state_->scan_idx_to_hash_pos_.emplace(level_idx, matching_row_index);
1194  CHECK(it_ok.second);
1195  return matching_row_index;
1196 }
1197 
1198 void Executor::codegenJoinLoops(const std::vector<JoinLoop>& join_loops,
1199  const RelAlgExecutionUnit& ra_exe_unit,
1200  GroupByAndAggregate& group_by_and_aggregate,
1201  llvm::Function* query_func,
1202  llvm::BasicBlock* entry_bb,
1204  const CompilationOptions& co,
1205  const ExecutionOptions& eo) {
1206  AUTOMATIC_IR_METADATA(cgen_state_.get());
1207  const auto exit_bb =
1208  llvm::BasicBlock::Create(cgen_state_->context_, "exit", cgen_state_->current_func_);
1209  cgen_state_->ir_builder_.SetInsertPoint(exit_bb);
1210  cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
1211  cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1212  CodeGenerator code_generator(this);
1213 
1214  llvm::BasicBlock* loops_entry_bb{nullptr};
1215  auto has_range_join =
1216  std::any_of(join_loops.begin(), join_loops.end(), [](const auto& join_loop) {
1217  return join_loop.kind() == JoinLoopKind::MultiSet;
1218  });
1219  if (has_range_join) {
1220  CHECK_EQ(join_loops.size(), size_t(1));
1221  const auto element_count =
1222  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_), 9);
1223 
1224  auto compute_packed_offset = [](const int32_t x, const int32_t y) -> uint64_t {
1225  const uint64_t y_shifted = static_cast<uint64_t>(y) << 32;
1226  return y_shifted | static_cast<uint32_t>(x);
1227  };
1228 
1229  const auto values_arr = std::vector<llvm::Constant*>{
1230  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_), 0),
1231  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1232  compute_packed_offset(0, 1)),
1233  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1234  compute_packed_offset(0, -1)),
1235  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1236  compute_packed_offset(1, 0)),
1237  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1238  compute_packed_offset(1, 1)),
1239  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1240  compute_packed_offset(1, -1)),
1241  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1242  compute_packed_offset(-1, 0)),
1243  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1244  compute_packed_offset(-1, 1)),
1245  llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
1246  compute_packed_offset(-1, -1))};
1247 
1248  const auto constant_values_array = llvm::ConstantArray::get(
1249  get_int_array_type(64, 9, cgen_state_->context_), values_arr);
1250  CHECK(cgen_state_->module_);
1251  const auto values =
1252  new llvm::GlobalVariable(*cgen_state_->module_,
1253  get_int_array_type(64, 9, cgen_state_->context_),
1254  true,
1255  llvm::GlobalValue::LinkageTypes::InternalLinkage,
1256  constant_values_array);
1257  JoinLoop join_loop(
1260  [element_count, values](const std::vector<llvm::Value*>& v) {
1261  JoinLoopDomain domain{{0}};
1262  domain.element_count = element_count;
1263  domain.values_buffer = values;
1264  return domain;
1265  },
1266  nullptr,
1267  nullptr,
1268  nullptr,
1269  nullptr,
1270  "range_key_loop");
1271 
1272  loops_entry_bb = JoinLoop::codegen(
1273  {join_loop},
1274  [this,
1275  query_func,
1276  &query_mem_desc,
1277  &co,
1278  &eo,
1279  &group_by_and_aggregate,
1280  &join_loops,
1281  &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
1282  auto& builder = cgen_state_->ir_builder_;
1283 
1284  auto body_exit_bb =
1285  llvm::BasicBlock::Create(cgen_state_->context_,
1286  "range_key_inner_body_exit",
1287  builder.GetInsertBlock()->getParent());
1288 
1289  auto range_key_body_bb =
1290  llvm::BasicBlock::Create(cgen_state_->context_,
1291  "range_key_loop_body",
1292  builder.GetInsertBlock()->getParent());
1293  builder.SetInsertPoint(range_key_body_bb);
1294 
1295  const auto body_loops_entry_bb = JoinLoop::codegen(
1296  join_loops,
1297  [this,
1298  query_func,
1299  &query_mem_desc,
1300  &co,
1301  &eo,
1302  &group_by_and_aggregate,
1303  &join_loops,
1304  &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
1305  addJoinLoopIterator(prev_iters, join_loops.size());
1306  auto& builder = cgen_state_->ir_builder_;
1307  const auto loop_body_bb =
1308  llvm::BasicBlock::Create(builder.getContext(),
1309  "loop_body",
1310  builder.GetInsertBlock()->getParent());
1311  builder.SetInsertPoint(loop_body_bb);
1312  const bool can_return_error =
1313  compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1314  if (can_return_error || cgen_state_->needs_error_check_ ||
1315  eo.with_dynamic_watchdog || eo.allow_runtime_query_interrupt) {
1316  createErrorCheckControlFlow(query_func,
1317  eo.with_dynamic_watchdog,
1318  eo.allow_runtime_query_interrupt,
1319  join_loops,
1320  co.device_type,
1321  group_by_and_aggregate.query_infos_);
1322  }
1323  return loop_body_bb;
1324  },
1325  prev_iters.back(),
1326  body_exit_bb,
1327  cgen_state_.get());
1328 
1329  builder.SetInsertPoint(range_key_body_bb);
1330  cgen_state_->ir_builder_.CreateBr(body_loops_entry_bb);
1331 
1332  builder.SetInsertPoint(body_exit_bb);
1333  return range_key_body_bb;
1334  },
1335  code_generator.posArg(nullptr),
1336  exit_bb,
1337  cgen_state_.get());
1338  } else {
1339  loops_entry_bb = JoinLoop::codegen(
1340  join_loops,
1341  /*body_codegen=*/
1342  [this,
1343  query_func,
1344  &query_mem_desc,
1345  &co,
1346  &eo,
1347  &group_by_and_aggregate,
1348  &join_loops,
1349  &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
1350  AUTOMATIC_IR_METADATA(cgen_state_.get());
1351  addJoinLoopIterator(prev_iters, join_loops.size());
1352  auto& builder = cgen_state_->ir_builder_;
1353  const auto loop_body_bb = llvm::BasicBlock::Create(
1354  builder.getContext(), "loop_body", builder.GetInsertBlock()->getParent());
1355  builder.SetInsertPoint(loop_body_bb);
1356  const bool can_return_error =
1357  compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1358  if (can_return_error || cgen_state_->needs_error_check_ ||
1360  createErrorCheckControlFlow(query_func,
1363  join_loops,
1364  co.device_type,
1365  group_by_and_aggregate.query_infos_);
1366  }
1367  return loop_body_bb;
1368  },
1369  /*outer_iter=*/code_generator.posArg(nullptr),
1370  exit_bb,
1371  cgen_state_.get());
1372  }
1373  CHECK(loops_entry_bb);
1374  cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1375  cgen_state_->ir_builder_.CreateBr(loops_entry_bb);
1376 }
1377 
1379  Analyzer::Expr* group_by_col,
1380  const size_t col_width,
1381  const CompilationOptions& co,
1382  const bool translate_null_val,
1383  const int64_t translated_null_val,
1384  DiamondCodegen& diamond_codegen,
1385  std::stack<llvm::BasicBlock*>& array_loops,
1386  const bool thread_mem_shared) {
1387  AUTOMATIC_IR_METADATA(cgen_state_.get());
1388  CHECK_GE(col_width, sizeof(int32_t));
1389  CodeGenerator code_generator(this);
1390  auto group_key = code_generator.codegen(group_by_col, true, co).front();
1391  auto key_to_cache = group_key;
1392  if (dynamic_cast<Analyzer::UOper*>(group_by_col) &&
1393  static_cast<Analyzer::UOper*>(group_by_col)->get_optype() == kUNNEST) {
1394  auto preheader = cgen_state_->ir_builder_.GetInsertBlock();
1395  auto array_loop_head = llvm::BasicBlock::Create(cgen_state_->context_,
1396  "array_loop_head",
1397  cgen_state_->current_func_,
1398  preheader->getNextNode());
1399  diamond_codegen.setFalseTarget(array_loop_head);
1400  const auto ret_ty = get_int_type(32, cgen_state_->context_);
1401  auto array_idx_ptr = cgen_state_->ir_builder_.CreateAlloca(ret_ty);
1402  CHECK(array_idx_ptr);
1403  cgen_state_->ir_builder_.CreateStore(cgen_state_->llInt(int32_t(0)), array_idx_ptr);
1404  const auto arr_expr = static_cast<Analyzer::UOper*>(group_by_col)->get_operand();
1405  const auto& array_ti = arr_expr->get_type_info();
1406  CHECK(array_ti.is_array());
1407  const auto& elem_ti = array_ti.get_elem_type();
1408  auto array_len =
1409  (array_ti.get_size() > 0)
1410  ? cgen_state_->llInt(array_ti.get_size() / elem_ti.get_size())
1411  : cgen_state_->emitExternalCall(
1412  "array_size",
1413  ret_ty,
1414  {group_key,
1415  code_generator.posArg(arr_expr),
1416  cgen_state_->llInt(log2_bytes(elem_ti.get_logical_size()))});
1417  cgen_state_->ir_builder_.CreateBr(array_loop_head);
1418  cgen_state_->ir_builder_.SetInsertPoint(array_loop_head);
1419  CHECK(array_len);
1420  auto array_idx = cgen_state_->ir_builder_.CreateLoad(
1421  array_idx_ptr->getType()->getPointerElementType(), array_idx_ptr);
1422  auto bound_check = cgen_state_->ir_builder_.CreateICmp(
1423  llvm::ICmpInst::ICMP_SLT, array_idx, array_len);
1424  auto array_loop_body = llvm::BasicBlock::Create(
1425  cgen_state_->context_, "array_loop_body", cgen_state_->current_func_);
1426  cgen_state_->ir_builder_.CreateCondBr(
1427  bound_check,
1428  array_loop_body,
1429  array_loops.empty() ? diamond_codegen.orig_cond_false_ : array_loops.top());
1430  cgen_state_->ir_builder_.SetInsertPoint(array_loop_body);
1431  cgen_state_->ir_builder_.CreateStore(
1432  cgen_state_->ir_builder_.CreateAdd(array_idx, cgen_state_->llInt(int32_t(1))),
1433  array_idx_ptr);
1434  auto array_at_fname = "array_at_" + numeric_type_name(elem_ti);
1435  if (array_ti.get_size() < 0) {
1436  if (array_ti.get_notnull()) {
1437  array_at_fname = "notnull_" + array_at_fname;
1438  }
1439  array_at_fname = "varlen_" + array_at_fname;
1440  }
1441  const auto ar_ret_ty =
1442  elem_ti.is_fp()
1443  ? (elem_ti.get_type() == kDOUBLE
1444  ? llvm::Type::getDoubleTy(cgen_state_->context_)
1445  : llvm::Type::getFloatTy(cgen_state_->context_))
1446  : get_int_type(elem_ti.get_logical_size() * 8, cgen_state_->context_);
1447  group_key = cgen_state_->emitExternalCall(
1448  array_at_fname,
1449  ar_ret_ty,
1450  {group_key, code_generator.posArg(arr_expr), array_idx});
1452  elem_ti, isArchMaxwell(co.device_type), thread_mem_shared)) {
1453  key_to_cache = spillDoubleElement(group_key, ar_ret_ty);
1454  } else {
1455  key_to_cache = group_key;
1456  }
1457  CHECK(array_loop_head);
1458  array_loops.push(array_loop_head);
1459  }
1460  cgen_state_->group_by_expr_cache_.push_back(key_to_cache);
1461  llvm::Value* orig_group_key{nullptr};
1462  if (translate_null_val) {
1463  const std::string translator_func_name(
1464  col_width == sizeof(int32_t) ? "translate_null_key_i32_" : "translate_null_key_");
1465  const auto& ti = group_by_col->get_type_info();
1466  const auto key_type = get_int_type(ti.get_logical_size() * 8, cgen_state_->context_);
1467  orig_group_key = group_key;
1468  group_key = cgen_state_->emitCall(
1469  translator_func_name + numeric_type_name(ti),
1470  {group_key,
1471  static_cast<llvm::Value*>(
1472  llvm::ConstantInt::get(key_type, inline_int_null_val(ti))),
1473  static_cast<llvm::Value*>(llvm::ConstantInt::get(
1474  llvm::Type::getInt64Ty(cgen_state_->context_), translated_null_val))});
1475  }
1476  group_key = cgen_state_->ir_builder_.CreateBitCast(
1477  cgen_state_->castToTypeIn(group_key, col_width * 8),
1478  get_int_type(col_width * 8, cgen_state_->context_));
1479  if (orig_group_key) {
1480  orig_group_key = cgen_state_->ir_builder_.CreateBitCast(
1481  cgen_state_->castToTypeIn(orig_group_key, col_width * 8),
1482  get_int_type(col_width * 8, cgen_state_->context_));
1483  }
1484  return {group_key, orig_group_key};
1485 }
1486 
1488  Executor* executor,
1489  llvm::Value* nullable_lv,
1490  const SQLTypeInfo& nullable_ti,
1491  const std::string& name)
1492  : cgen_state(cgen_state), name(name) {
1493  AUTOMATIC_IR_METADATA(cgen_state);
1494  CHECK(nullable_ti.is_number() || nullable_ti.is_time() || nullable_ti.is_boolean() ||
1495  nullable_ti.is_dict_encoded_string());
1496 
1497  llvm::Value* is_null_lv{nullptr};
1498  if (nullable_ti.is_fp()) {
1499  is_null_lv = cgen_state->ir_builder_.CreateFCmp(
1500  llvm::FCmpInst::FCMP_OEQ, nullable_lv, cgen_state->inlineFpNull(nullable_ti));
1501  } else if (nullable_ti.is_boolean() &&
1502  nullable_lv->getType()->getIntegerBitWidth() == 1) {
1503  is_null_lv = cgen_state->ir_builder_.CreateICmp(
1504  llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->llBool(true));
1505  } else {
1506  is_null_lv = cgen_state->ir_builder_.CreateICmp(
1507  llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->inlineIntNull(nullable_ti));
1508  }
1509  CHECK(is_null_lv);
1510  null_check =
1511  std::make_unique<DiamondCodegen>(is_null_lv, executor, false, name, nullptr, false);
1512 
1513  // generate a phi node depending on whether we got a null or not
1514  nullcheck_bb = llvm::BasicBlock::Create(
1515  cgen_state->context_, name + "_bb", cgen_state->current_func_);
1516 
1517  // update the blocks created by diamond codegen to point to the newly created phi
1518  // block
1519  cgen_state->ir_builder_.SetInsertPoint(null_check->cond_true_);
1520  cgen_state->ir_builder_.CreateBr(nullcheck_bb);
1521  cgen_state->ir_builder_.SetInsertPoint(null_check->cond_false_);
1522 }
1523 
1524 llvm::Value* CodeGenerator::NullCheckCodegen::finalize(llvm::Value* null_lv,
1525  llvm::Value* notnull_lv) {
1526  AUTOMATIC_IR_METADATA(cgen_state);
1527  CHECK(null_check);
1528  cgen_state->ir_builder_.CreateBr(nullcheck_bb);
1529 
1530  CHECK_EQ(null_lv->getType(), notnull_lv->getType());
1531 
1532  cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1533  nullcheck_value =
1534  cgen_state->ir_builder_.CreatePHI(null_lv->getType(), 2, name + "_value");
1535  nullcheck_value->addIncoming(notnull_lv, null_check->cond_false_);
1536  nullcheck_value->addIncoming(null_lv, null_check->cond_true_);
1537 
1538  null_check.reset(nullptr);
1539  cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1540  return nullcheck_value;
1541 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
bool g_enable_left_join_filter_hoisting
Definition: Execute.cpp:101
NullCheckCodegen(CgenState *cgen_state, Executor *executor, llvm::Value *nullable_lv, const SQLTypeInfo &nullable_ti, const std::string &name="")
Definition: IRCodegen.cpp:1487
void codegenJoinLoops(const std::vector< JoinLoop > &join_loops, const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, llvm::Function *query_func, llvm::BasicBlock *entry_bb, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const ExecutionOptions &eo)
Definition: IRCodegen.cpp:1198
#define IS_LOGIC(X)
Definition: sqldefs.h:61
const Expr * get_partition_count() const
Definition: Analyzer.h:1201
#define NULL_DOUBLE
JoinType
Definition: sqldefs.h:172
std::vector< llvm::Value * > outer_join_match_found_per_level_
Definition: CgenState.h:395
bool is_constant_expr() const
Definition: Analyzer.h:1246
std::unordered_map< size_t, std::vector< std::shared_ptr< Analyzer::Expr > > > left_join_non_hashtable_quals_
Definition: PlanState.h:61
llvm::Value * codegenConstantWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
Definition: IRCodegen.cpp:364
llvm::BasicBlock * nullcheck_bb
llvm::Value * element_count
Definition: JoinLoop.h:46
llvm::Value * values_buffer
Definition: JoinLoop.h:49
#define IS_EQUIVALENCE(X)
Definition: sqldefs.h:69
llvm::Value * codegenArith(const Analyzer::BinOper *, const CompilationOptions &)
CgenState * cgen_state_
GroupColLLVMValue groupByColumnCodegen(Analyzer::Expr *group_by_col, const size_t col_width, const CompilationOptions &, const bool translate_null_val, const int64_t translated_null_val, DiamondCodegen &, std::stack< llvm::BasicBlock * > &, const bool thread_mem_shared)
Definition: IRCodegen.cpp:1378
bool is_fp() const
Definition: sqltypes.h:571
llvm::IRBuilder ir_builder_
Definition: CgenState.h:384
std::function< llvm::BasicBlock *(llvm::BasicBlock *, llvm::BasicBlock *, const std::string &, llvm::Function *, CgenState *)> HoistedFiltersCallback
Definition: JoinLoop.h:61
llvm::Value * posArg(const Analyzer::Expr *) const
Definition: ColumnIR.cpp:585
std::string join(T const &container, std::string const &delim)
std::vector< InputDescriptor > input_descs
#define UNREACHABLE()
Definition: Logger.h:338
#define CHECK_GE(x, y)
Definition: Logger.h:306
bool need_patch_unnest_double(const SQLTypeInfo &ti, const bool is_maxwell, const bool mem_shared)
Definition: sqldefs.h:48
llvm::ConstantInt * llBool(const bool v) const
Definition: CgenState.h:263
virtual std::vector< llvm::Value * > codegenColumn(const Analyzer::ColumnVar *, const bool fetch_column, const CompilationOptions &)
Definition: ColumnIR.cpp:94
void set_constant_expr() const
Definition: Analyzer.h:1245
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:90
llvm::Value * codegenArrayAt(const Analyzer::BinOper *, const CompilationOptions &)
Definition: ArrayIR.cpp:26
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
void setFalseTarget(llvm::BasicBlock *cond_false)
QualsConjunctiveForm qual_to_conjunctive_form(const std::shared_ptr< Analyzer::Expr > qual_expr)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
bool is_number() const
Definition: sqltypes.h:574
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::vector< llvm::Value * > codegenGeoBinOper(const Analyzer::GeoBinOper *, const CompilationOptions &)
Definition: GeoIR.cpp:242
bool is_time() const
Definition: sqltypes.h:577
std::shared_ptr< HashJoin > hash_table
Definition: Execute.h:1234
double get_bound_val(const Analyzer::Expr *bound_expr) const
Definition: Analyzer.cpp:3903
std::string to_string(char const *&&v)
llvm::Function * row_func_
Definition: CgenState.h:374
llvm::Value * codegenIsNull(const Analyzer::UOper *, const CompilationOptions &)
Definition: LogicalIR.cpp:380
SQLOps get_optype() const
Definition: Analyzer.h:452
std::vector< llvm::Value * > codegenGeoExpr(const Analyzer::GeoExpr *, const CompilationOptions &)
Definition: GeoIR.cpp:95
llvm::LLVMContext & context_
Definition: CgenState.h:382
llvm::Function * current_func_
Definition: CgenState.h:376
llvm::Value * get_arg_by_name(llvm::Function *func, const std::string &name)
Definition: Execute.h:168
std::set< shared::TableKey > visitFunctionOper(const Analyzer::FunctionOper *func_expr) const final
Definition: IRCodegen.cpp:821
#define INJECT_TIMER(DESC)
Definition: measure.h:96
#define CHECK_NE(x, y)
Definition: Logger.h:302
const JoinQualsPerNestingLevel join_quals
std::vector< llvm::Value * > codegenGeoUOper(const Analyzer::GeoUOper *, const CompilationOptions &)
Definition: GeoIR.cpp:162
llvm::ConstantInt * inlineIntNull(const SQLTypeInfo &)
Definition: CgenState.cpp:65
TableIdToNodeMap table_id_to_node_map
llvm::Value * codegenWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
Definition: IRCodegen.cpp:437
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
Definition: CastIR.cpp:427
llvm::Value * codegenFunctionOper(const Analyzer::FunctionOper *, const CompilationOptions &)
Executor * executor_
static llvm::BasicBlock * codegen(const std::vector< JoinLoop > &join_loops, const std::function< llvm::BasicBlock *(const std::vector< llvm::Value * > &)> &body_codegen, llvm::Value *outer_iter, llvm::BasicBlock *exit_bb, CgenState *cgen_state)
Definition: JoinLoop.cpp:50
void add_qualifier_to_execution_unit(RelAlgExecutionUnit &ra_exe_unit, const std::shared_ptr< Analyzer::Expr > &qual)
Definition: IRCodegen.cpp:535
const std::vector< InputTableInfo > & query_infos_
Definition: PlanState.h:65
bool needs_error_check_
Definition: CgenState.h:405
bool is_boolean() const
Definition: sqltypes.h:580
ExpressionRange getExpressionRange(const Analyzer::BinOper *expr, const std::vector< InputTableInfo > &query_infos, const Executor *, boost::optional< std::list< std::shared_ptr< Analyzer::Expr >>> simple_quals)
std::vector< llvm::Value * > codegenArrayExpr(const Analyzer::ArrayExpr *, const CompilationOptions &)
Definition: ArrayIR.cpp:97
#define AUTOMATIC_IR_METADATA(CGENSTATE)
llvm::BasicBlock * orig_cond_false_
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:79
llvm::Value * codegenUMinus(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
Definition: CgenState.cpp:217
llvm::Value * slot_lookup_result
Definition: JoinLoop.h:47
ExecutorDeviceType device_type
Definition: sqldefs.h:33
PlanState * plan_state_
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
Definition: IRCodegen.cpp:30
#define CHECK_LT(x, y)
Definition: Logger.h:303
const std::vector< InputTableInfo > & query_infos_
Definition: sqldefs.h:71
Expression class for string functions The &quot;arg&quot; constructor parameter must be an expression that reso...
Definition: Analyzer.h:1601
std::set< shared::TableKey > aggregateResult(const std::set< shared::TableKey > &aggregate, const std::set< shared::TableKey > &next_result) const final
Definition: IRCodegen.cpp:840
std::shared_ptr< HashJoin > buildCurrentLevelHashTable(const JoinCondition &current_level_join_conditions, size_t level_idx, RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache, std::vector< std::string > &fail_reasons)
Definition: IRCodegen.cpp:1021
#define IS_ARITHMETIC(X)
Definition: sqldefs.h:62
bool isHintRegistered(const QueryHint hint) const
Definition: QueryHint.h:366
const Expr * get_arg() const
Definition: Analyzer.h:962
std::set< shared::TableKey > visitBinOper(const Analyzer::BinOper *bin_oper) const final
Definition: IRCodegen.cpp:830
std::unordered_map< shared::TableKey, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
std::set< shared::TableKey > visitUOper(const Analyzer::UOper *u_oper) const final
Definition: IRCodegen.cpp:836
llvm::StructType * createStringViewStructType()
int32_t get_partition_count_val() const
Definition: Analyzer.cpp:3913
static const int32_t ERR_WIDTH_BUCKET_INVALID_ARGUMENT
Definition: Execute.h:1628
llvm::Value * toBool(llvm::Value *)
Definition: LogicalIR.cpp:343
std::vector< llvm::Value * > codegenGeoColumnVar(const Analyzer::GeoColumnVar *, const bool fetch_columns, const CompilationOptions &co)
Definition: GeoIR.cpp:52
llvm::Value * codegenFunctionOperWithCustomTypeHandling(const Analyzer::FunctionOperWithCustomTypeHandling *, const CompilationOptions &)
llvm::Value * codegenCmp(const Analyzer::BinOper *, const CompilationOptions &)
Definition: CompareIR.cpp:230
std::list< std::shared_ptr< Analyzer::Expr > > getSimpleQuals() const
Definition: PlanState.h:97
const Expr * get_target_value() const
Definition: Analyzer.h:1198
std::list< std::shared_ptr< Analyzer::Expr > > quals
llvm::ConstantInt * llInt(const T v) const
Definition: CgenState.h:249
llvm::Value * codegenUnnest(const Analyzer::UOper *, const CompilationOptions &)
Definition: ArrayIR.cpp:20
llvm::Value * addJoinLoopIterator(const std::vector< llvm::Value * > &prev_iters, const size_t level_idx)
Definition: IRCodegen.cpp:1180
std::list< std::shared_ptr< Analyzer::Expr > > quals
RegisteredQueryHint query_hint
llvm::Value * finalize(llvm::Value *null_lv, llvm::Value *notnull_lv)
Definition: IRCodegen.cpp:1524
#define CHECK(condition)
Definition: Logger.h:291
bool can_skip_out_of_bound_check() const
Definition: Analyzer.h:1243
llvm::Value * codegenLogical(const Analyzer::BinOper *, const CompilationOptions &)
Definition: LogicalIR.cpp:298
void check_if_loop_join_is_allowed(RelAlgExecutionUnit &ra_exe_unit, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, const size_t level_idx, const std::string &fail_reason)
Definition: IRCodegen.cpp:545
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
llvm::ConstantInt * ll_bool(const bool v, llvm::LLVMContext &context)
size_t loop_join_inner_table_max_num_rows
Definition: QueryHint.h:340
llvm::Value * codegenCast(const Analyzer::UOper *, const CompilationOptions &)
Definition: CastIR.cpp:21
uint32_t log2_bytes(const uint32_t bytes)
Definition: Execute.h:198
std::string numeric_type_name(const SQLTypeInfo &ti)
Definition: Execute.h:230
bool is_dict_encoded_string() const
Definition: sqltypes.h:641
Definition: sqltypes.h:72
void skip_out_of_bound_check() const
Definition: Analyzer.h:1244
void redeclareFilterFunction()
Definition: IRCodegen.cpp:1081
bool any_of(std::vector< Analyzer::Expr * > const &target_exprs)
const Expr * get_lower_bound() const
Definition: Analyzer.h:1199
std::vector< JoinLoop > buildJoinLoops(RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache)
Definition: IRCodegen.cpp:605
std::function< llvm::Value *(const std::vector< llvm::Value * > &, llvm::Value *)> buildIsDeletedCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const CompilationOptions &co)
Definition: IRCodegen.cpp:962
string name
Definition: setup.in.py:72
JoinLoop::HoistedFiltersCallback buildHoistLeftHandSideFiltersCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const shared::TableKey &inner_table_key, const CompilationOptions &co)
Definition: IRCodegen.cpp:853
const Expr * get_upper_bound() const
Definition: Analyzer.h:1200
Definition: Datum.h:69
llvm::ArrayType * get_int_array_type(int const width, int count, llvm::LLVMContext &context)
Definition: sqldefs.h:38
SQLOps get_optype() const
Definition: Analyzer.h:383
#define VLOG(n)
Definition: Logger.h:388
std::unique_ptr< DiamondCodegen > null_check
std::set< shared::TableKey > visitColumnVar(const Analyzer::ColumnVar *col_expr) const final
Definition: IRCodegen.cpp:816
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals
#define IS_COMPARISON(X)
Definition: sqldefs.h:58
double doubleval
Definition: Datum.h:76
HashTableBuildDagMap hash_table_build_plan_dag
llvm::ConstantFP * inlineFpNull(const SQLTypeInfo &)
Definition: CgenState.cpp:104
void check_valid_join_qual(std::shared_ptr< Analyzer::BinOper > &bin_oper)
Definition: IRCodegen.cpp:586
Executor * executor() const
size_t get_loop_join_size(const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
Definition: Execute.cpp:1806
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr(const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count)