OmniSciDB  06b3bd477c
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
IRCodegen.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "../Parser/ParserNode.h"
18 #include "CodeGenerator.h"
19 #include "Execute.h"
20 #include "ExternalExecutor.h"
21 #include "MaxwellCodegenPatch.h"
22 #include "RelAlgTranslator.h"
23 
24 // Driver methods for the IR generation.
25 
26 std::vector<llvm::Value*> CodeGenerator::codegen(const Analyzer::Expr* expr,
27  const bool fetch_columns,
28  const CompilationOptions& co) {
29  if (!expr) {
30  return {posArg(expr)};
31  }
32  auto bin_oper = dynamic_cast<const Analyzer::BinOper*>(expr);
33  if (bin_oper) {
34  return {codegen(bin_oper, co)};
35  }
36  auto u_oper = dynamic_cast<const Analyzer::UOper*>(expr);
37  if (u_oper) {
38  return {codegen(u_oper, co)};
39  }
40  auto col_var = dynamic_cast<const Analyzer::ColumnVar*>(expr);
41  if (col_var) {
42  return codegenColumn(col_var, fetch_columns, co);
43  }
44  auto constant = dynamic_cast<const Analyzer::Constant*>(expr);
45  if (constant) {
46  const auto& ti = constant->get_type_info();
47  if (ti.get_type() == kNULLT) {
48  throw std::runtime_error(
49  "NULL type literals are not currently supported in this context.");
50  }
51  if (constant->get_is_null()) {
52  return {ti.is_fp()
53  ? static_cast<llvm::Value*>(executor_->cgen_state_->inlineFpNull(ti))
54  : static_cast<llvm::Value*>(executor_->cgen_state_->inlineIntNull(ti))};
55  }
56  if (ti.get_compression() == kENCODING_DICT) {
57  // The dictionary encoding case should be handled by the parent expression
58  // (cast, for now), here is too late to know the dictionary id if not already set
59  CHECK_NE(ti.get_comp_param(), 0);
60  return {codegen(constant, ti.get_compression(), ti.get_comp_param(), co)};
61  }
62  return {codegen(constant, ti.get_compression(), 0, co)};
63  }
64  auto case_expr = dynamic_cast<const Analyzer::CaseExpr*>(expr);
65  if (case_expr) {
66  return {codegen(case_expr, co)};
67  }
68  auto extract_expr = dynamic_cast<const Analyzer::ExtractExpr*>(expr);
69  if (extract_expr) {
70  return {codegen(extract_expr, co)};
71  }
72  auto dateadd_expr = dynamic_cast<const Analyzer::DateaddExpr*>(expr);
73  if (dateadd_expr) {
74  return {codegen(dateadd_expr, co)};
75  }
76  auto datediff_expr = dynamic_cast<const Analyzer::DatediffExpr*>(expr);
77  if (datediff_expr) {
78  return {codegen(datediff_expr, co)};
79  }
80  auto datetrunc_expr = dynamic_cast<const Analyzer::DatetruncExpr*>(expr);
81  if (datetrunc_expr) {
82  return {codegen(datetrunc_expr, co)};
83  }
84  auto charlength_expr = dynamic_cast<const Analyzer::CharLengthExpr*>(expr);
85  if (charlength_expr) {
86  return {codegen(charlength_expr, co)};
87  }
88  auto keyforstring_expr = dynamic_cast<const Analyzer::KeyForStringExpr*>(expr);
89  if (keyforstring_expr) {
90  return {codegen(keyforstring_expr, co)};
91  }
92  auto sample_ratio_expr = dynamic_cast<const Analyzer::SampleRatioExpr*>(expr);
93  if (sample_ratio_expr) {
94  return {codegen(sample_ratio_expr, co)};
95  }
96  auto lower_expr = dynamic_cast<const Analyzer::LowerExpr*>(expr);
97  if (lower_expr) {
98  return {codegen(lower_expr, co)};
99  }
100  auto cardinality_expr = dynamic_cast<const Analyzer::CardinalityExpr*>(expr);
101  if (cardinality_expr) {
102  return {codegen(cardinality_expr, co)};
103  }
104  auto like_expr = dynamic_cast<const Analyzer::LikeExpr*>(expr);
105  if (like_expr) {
106  return {codegen(like_expr, co)};
107  }
108  auto regexp_expr = dynamic_cast<const Analyzer::RegexpExpr*>(expr);
109  if (regexp_expr) {
110  return {codegen(regexp_expr, co)};
111  }
112  auto likelihood_expr = dynamic_cast<const Analyzer::LikelihoodExpr*>(expr);
113  if (likelihood_expr) {
114  return {codegen(likelihood_expr->get_arg(), fetch_columns, co)};
115  }
116  auto in_expr = dynamic_cast<const Analyzer::InValues*>(expr);
117  if (in_expr) {
118  return {codegen(in_expr, co)};
119  }
120  auto in_integer_set_expr = dynamic_cast<const Analyzer::InIntegerSet*>(expr);
121  if (in_integer_set_expr) {
122  return {codegen(in_integer_set_expr, co)};
123  }
124  auto function_oper_with_custom_type_handling_expr =
125  dynamic_cast<const Analyzer::FunctionOperWithCustomTypeHandling*>(expr);
126  if (function_oper_with_custom_type_handling_expr) {
128  function_oper_with_custom_type_handling_expr, co)};
129  }
130  auto array_oper_expr = dynamic_cast<const Analyzer::ArrayExpr*>(expr);
131  if (array_oper_expr) {
132  return {codegenArrayExpr(array_oper_expr, co)};
133  }
134  auto geo_uop = dynamic_cast<const Analyzer::GeoUOper*>(expr);
135  if (geo_uop) {
136  return {codegenGeoUOper(geo_uop, co)};
137  }
138  auto geo_binop = dynamic_cast<const Analyzer::GeoBinOper*>(expr);
139  if (geo_binop) {
140  return {codegenGeoBinOper(geo_binop, co)};
141  }
142  auto function_oper_expr = dynamic_cast<const Analyzer::FunctionOper*>(expr);
143  if (function_oper_expr) {
144  return {codegenFunctionOper(function_oper_expr, co)};
145  }
146  if (dynamic_cast<const Analyzer::OffsetInFragment*>(expr)) {
147  return {posArg(nullptr)};
148  }
149  if (dynamic_cast<const Analyzer::WindowFunction*>(expr)) {
150  throw NativeExecutionError("Window expression not supported in this context");
151  }
152  abort();
153 }
154 
155 llvm::Value* CodeGenerator::codegen(const Analyzer::BinOper* bin_oper,
156  const CompilationOptions& co) {
157  const auto optype = bin_oper->get_optype();
158  if (IS_ARITHMETIC(optype)) {
159  return codegenArith(bin_oper, co);
160  }
161  if (IS_COMPARISON(optype)) {
162  return codegenCmp(bin_oper, co);
163  }
164  if (IS_LOGIC(optype)) {
165  return codegenLogical(bin_oper, co);
166  }
167  if (optype == kARRAY_AT) {
168  return codegenArrayAt(bin_oper, co);
169  }
170  abort();
171 }
172 
173 llvm::Value* CodeGenerator::codegen(const Analyzer::UOper* u_oper,
174  const CompilationOptions& co) {
175  const auto optype = u_oper->get_optype();
176  switch (optype) {
177  case kNOT: {
178  return codegenLogical(u_oper, co);
179  }
180  case kCAST: {
181  return codegenCast(u_oper, co);
182  }
183  case kUMINUS: {
184  return codegenUMinus(u_oper, co);
185  }
186  case kISNULL: {
187  return codegenIsNull(u_oper, co);
188  }
189  case kUNNEST:
190  return codegenUnnest(u_oper, co);
191  default:
192  abort();
193  }
194 }
195 
197  const CompilationOptions& co) {
198  auto input_expr = expr->get_arg();
199  CHECK(input_expr);
200 
201  auto double_lv = codegen(input_expr, true, co);
202  CHECK_EQ(size_t(1), double_lv.size());
203 
204  std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
205  const bool is_nullable = !input_expr->get_type_info().get_notnull();
206  if (is_nullable) {
207  nullcheck_codegen = std::make_unique<NullCheckCodegen>(cgen_state_,
208  executor(),
209  double_lv.front(),
210  input_expr->get_type_info(),
211  "sample_ratio_nullcheck");
212  }
213  CHECK_EQ(input_expr->get_type_info().get_type(), kDOUBLE);
214  std::vector<llvm::Value*> args{double_lv[0], posArg(nullptr)};
215  auto ret = cgen_state_->emitCall("sample_ratio", args);
216  if (nullcheck_codegen) {
217  ret = nullcheck_codegen->finalize(ll_bool(false, cgen_state_->context_), ret);
218  }
219  return ret;
220 }
221 
222 namespace {
223 
225  const std::shared_ptr<Analyzer::Expr>& qual) {
226  const auto qual_cf = qual_to_conjunctive_form(qual);
227  ra_exe_unit.simple_quals.insert(ra_exe_unit.simple_quals.end(),
228  qual_cf.simple_quals.begin(),
229  qual_cf.simple_quals.end());
230  ra_exe_unit.quals.insert(
231  ra_exe_unit.quals.end(), qual_cf.quals.begin(), qual_cf.quals.end());
232 }
233 
235  const ExecutionOptions& eo,
236  const std::vector<InputTableInfo>& query_infos,
237  const size_t level_idx,
238  const std::string& fail_reason) {
239  if (eo.allow_loop_joins) {
240  return;
241  }
242  if (level_idx + 1 != ra_exe_unit.join_quals.size()) {
243  throw std::runtime_error(
244  "Hash join failed, reason(s): " + fail_reason +
245  " | Cannot fall back to loop join for intermediate join quals");
246  }
247  if (!is_trivial_loop_join(query_infos, ra_exe_unit)) {
248  throw std::runtime_error(
249  "Hash join failed, reason(s): " + fail_reason +
250  " | Cannot fall back to loop join for non-trivial inner table size");
251  }
252 }
253 
254 } // namespace
255 
256 std::vector<JoinLoop> Executor::buildJoinLoops(
257  RelAlgExecutionUnit& ra_exe_unit,
258  const CompilationOptions& co,
259  const ExecutionOptions& eo,
260  const std::vector<InputTableInfo>& query_infos,
261  ColumnCacheMap& column_cache) {
262  INJECT_TIMER(buildJoinLoops);
263  std::vector<JoinLoop> join_loops;
264  for (size_t level_idx = 0, current_hash_table_idx = 0;
265  level_idx < ra_exe_unit.join_quals.size();
266  ++level_idx) {
267  const auto& current_level_join_conditions = ra_exe_unit.join_quals[level_idx];
268  std::vector<std::string> fail_reasons;
269  const auto build_cur_level_hash_table = [&]() {
270  if (current_level_join_conditions.quals.size() > 1) {
271  const auto first_qual = *current_level_join_conditions.quals.begin();
272  auto qual_bin_oper =
273  std::dynamic_pointer_cast<const Analyzer::BinOper>(first_qual);
274  if (qual_bin_oper && qual_bin_oper->is_overlaps_oper() &&
275  current_level_join_conditions.type == JoinType::LEFT) {
276  JoinCondition join_condition{{first_qual}, current_level_join_conditions.type};
277 
278  return buildCurrentLevelHashTable(
279  join_condition, ra_exe_unit, co, query_infos, column_cache, fail_reasons);
280  }
281  }
282  return buildCurrentLevelHashTable(current_level_join_conditions,
283  ra_exe_unit,
284  co,
285  query_infos,
286  column_cache,
287  fail_reasons);
288  };
289  const auto current_level_hash_table = build_cur_level_hash_table();
290  const auto found_outer_join_matches_cb =
291  [this, level_idx](llvm::Value* found_outer_join_matches) {
295  found_outer_join_matches;
296  };
297  const auto is_deleted_cb = buildIsDeletedCb(ra_exe_unit, level_idx, co);
298  const auto outer_join_condition_multi_quals_cb =
299  [this, level_idx, &co, &current_level_join_conditions](
300  const std::vector<llvm::Value*>& prev_iters) {
301  // The values generated for the match path don't dominate all uses
302  // since on the non-match path nulls are generated. Reset the cache
303  // once the condition is generated to avoid incorrect reuse.
304  FetchCacheAnchor anchor(cgen_state_.get());
305  addJoinLoopIterator(prev_iters, level_idx + 1);
306  llvm::Value* left_join_cond = cgen_state_->llBool(true);
307  CodeGenerator code_generator(this);
308  // Do not want to look at all quals! only 1..N quals (ignore first qual)
309  // Note(jclay): this may need to support cases larger than 2
310  // are there any?
311  if (current_level_join_conditions.quals.size() >= 2) {
312  auto qual_it = std::next(current_level_join_conditions.quals.begin(), 1);
313  for (; qual_it != current_level_join_conditions.quals.end();
314  std::advance(qual_it, 1)) {
315  left_join_cond = cgen_state_->ir_builder_.CreateAnd(
316  left_join_cond,
317  code_generator.toBool(
318  code_generator.codegen(qual_it->get(), true, co).front()));
319  }
320  }
321  return left_join_cond;
322  };
323  if (current_level_hash_table) {
324  if (current_level_hash_table->getHashType() == JoinHashTable::HashType::OneToOne) {
325  join_loops.emplace_back(
327  current_level_join_conditions.type,
328  [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
329  const std::vector<llvm::Value*>& prev_iters) {
330  addJoinLoopIterator(prev_iters, level_idx);
331  JoinLoopDomain domain{{0}};
332  domain.slot_lookup_result =
333  current_level_hash_table->codegenSlot(co, current_hash_table_idx);
334  return domain;
335  },
336  nullptr,
337  current_level_join_conditions.type == JoinType::LEFT
338  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
339  : nullptr,
340  is_deleted_cb);
341  } else {
342  join_loops.emplace_back(
343  /*kind=*/JoinLoopKind::Set,
344  /*type=*/current_level_join_conditions.type,
345  /*iteration_domain_codegen=*/
346  [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
347  const std::vector<llvm::Value*>& prev_iters) {
348  addJoinLoopIterator(prev_iters, level_idx);
349  JoinLoopDomain domain{{0}};
350  const auto matching_set = current_level_hash_table->codegenMatchingSet(
351  co, current_hash_table_idx);
352  domain.values_buffer = matching_set.elements;
353  domain.element_count = matching_set.count;
354  return domain;
355  },
356  /*outer_condition_match=*/
357  current_level_join_conditions.type == JoinType::LEFT
358  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
359  outer_join_condition_multi_quals_cb)
360  : nullptr,
361  /*found_outer_matches=*/current_level_join_conditions.type == JoinType::LEFT
362  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
363  : nullptr,
364  /*is_deleted=*/is_deleted_cb);
365  }
366  ++current_hash_table_idx;
367  } else {
368  const auto fail_reasons_str = current_level_join_conditions.quals.empty()
369  ? "No equijoin expression found"
370  : boost::algorithm::join(fail_reasons, " | ");
372  ra_exe_unit, eo, query_infos, level_idx, fail_reasons_str);
373  // Callback provided to the `JoinLoop` framework to evaluate the (outer) join
374  // condition.
375  VLOG(1) << "Unable to build hash table, falling back to loop join: "
376  << fail_reasons_str;
377  const auto outer_join_condition_cb =
378  [this, level_idx, &co, &current_level_join_conditions](
379  const std::vector<llvm::Value*>& prev_iters) {
380  // The values generated for the match path don't dominate all uses
381  // since on the non-match path nulls are generated. Reset the cache
382  // once the condition is generated to avoid incorrect reuse.
383  FetchCacheAnchor anchor(cgen_state_.get());
384  addJoinLoopIterator(prev_iters, level_idx + 1);
385  llvm::Value* left_join_cond = cgen_state_->llBool(true);
386  CodeGenerator code_generator(this);
387  for (auto expr : current_level_join_conditions.quals) {
388  left_join_cond = cgen_state_->ir_builder_.CreateAnd(
389  left_join_cond,
390  code_generator.toBool(
391  code_generator.codegen(expr.get(), true, co).front()));
392  }
393  return left_join_cond;
394  };
395  join_loops.emplace_back(
396  /*kind=*/JoinLoopKind::UpperBound,
397  /*type=*/current_level_join_conditions.type,
398  /*iteration_domain_codegen=*/
399  [this, level_idx](const std::vector<llvm::Value*>& prev_iters) {
400  addJoinLoopIterator(prev_iters, level_idx);
401  JoinLoopDomain domain{{0}};
402  const auto rows_per_scan_ptr = cgen_state_->ir_builder_.CreateGEP(
403  get_arg_by_name(cgen_state_->row_func_, "num_rows_per_scan"),
404  cgen_state_->llInt(int32_t(level_idx + 1)));
405  domain.upper_bound = cgen_state_->ir_builder_.CreateLoad(rows_per_scan_ptr,
406  "num_rows_per_scan");
407  return domain;
408  },
409  /*outer_condition_match=*/
410  current_level_join_conditions.type == JoinType::LEFT
411  ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
412  outer_join_condition_cb)
413  : nullptr,
414  /*found_outer_matches=*/
415  current_level_join_conditions.type == JoinType::LEFT
416  ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
417  : nullptr,
418  /*is_deleted=*/is_deleted_cb);
419  }
420  }
421  return join_loops;
422 }
423 
424 std::function<llvm::Value*(const std::vector<llvm::Value*>&, llvm::Value*)>
426  const size_t level_idx,
427  const CompilationOptions& co) {
428  if (!co.filter_on_deleted_column) {
429  return nullptr;
430  }
431  CHECK_LT(level_idx + 1, ra_exe_unit.input_descs.size());
432  const auto input_desc = ra_exe_unit.input_descs[level_idx + 1];
433  if (input_desc.getSourceType() != InputSourceType::TABLE) {
434  return nullptr;
435  }
436  const auto td = catalog_->getMetadataForTable(input_desc.getTableId());
437  CHECK(td);
438  const auto deleted_cd = catalog_->getDeletedColumnIfRowsDeleted(td);
439  if (!deleted_cd) {
440  return nullptr;
441  }
442  CHECK(deleted_cd->columnType.is_boolean());
443  const auto deleted_expr = makeExpr<Analyzer::ColumnVar>(deleted_cd->columnType,
444  input_desc.getTableId(),
445  deleted_cd->columnId,
446  input_desc.getNestLevel());
447  return [this, deleted_expr, level_idx, &co](const std::vector<llvm::Value*>& prev_iters,
448  llvm::Value* have_more_inner_rows) {
449  const auto matching_row_index = addJoinLoopIterator(prev_iters, level_idx + 1);
450  // Avoid fetching the deleted column from a position which is not valid.
451  // An invalid position can be returned by a one to one hash lookup (negative)
452  // or at the end of iteration over a set of matching values.
453  llvm::Value* is_valid_it{nullptr};
454  if (have_more_inner_rows) {
455  is_valid_it = have_more_inner_rows;
456  } else {
457  is_valid_it = cgen_state_->ir_builder_.CreateICmp(
458  llvm::ICmpInst::ICMP_SGE, matching_row_index, cgen_state_->llInt<int64_t>(0));
459  }
460  const auto it_valid_bb = llvm::BasicBlock::Create(
461  cgen_state_->context_, "it_valid", cgen_state_->row_func_);
462  const auto it_not_valid_bb = llvm::BasicBlock::Create(
463  cgen_state_->context_, "it_not_valid", cgen_state_->row_func_);
464  cgen_state_->ir_builder_.CreateCondBr(is_valid_it, it_valid_bb, it_not_valid_bb);
465  const auto row_is_deleted_bb = llvm::BasicBlock::Create(
466  cgen_state_->context_, "row_is_deleted", cgen_state_->row_func_);
467  cgen_state_->ir_builder_.SetInsertPoint(it_valid_bb);
468  CodeGenerator code_generator(this);
469  const auto row_is_deleted = code_generator.toBool(
470  code_generator.codegen(deleted_expr.get(), true, co).front());
471  cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
472  cgen_state_->ir_builder_.SetInsertPoint(it_not_valid_bb);
473  const auto row_is_deleted_default = cgen_state_->llBool(false);
474  cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
475  cgen_state_->ir_builder_.SetInsertPoint(row_is_deleted_bb);
476  auto row_is_deleted_or_default =
477  cgen_state_->ir_builder_.CreatePHI(row_is_deleted->getType(), 2);
478  row_is_deleted_or_default->addIncoming(row_is_deleted, it_valid_bb);
479  row_is_deleted_or_default->addIncoming(row_is_deleted_default, it_not_valid_bb);
480  return row_is_deleted_or_default;
481  };
482 }
483 
484 std::shared_ptr<JoinHashTableInterface> Executor::buildCurrentLevelHashTable(
485  const JoinCondition& current_level_join_conditions,
486  RelAlgExecutionUnit& ra_exe_unit,
487  const CompilationOptions& co,
488  const std::vector<InputTableInfo>& query_infos,
489  ColumnCacheMap& column_cache,
490  std::vector<std::string>& fail_reasons) {
491  if (current_level_join_conditions.type != JoinType::INNER &&
492  current_level_join_conditions.quals.size() > 1) {
493  fail_reasons.emplace_back("No equijoin expression found for outer join");
494  return nullptr;
495  }
496  std::shared_ptr<JoinHashTableInterface> current_level_hash_table;
497  for (const auto& join_qual : current_level_join_conditions.quals) {
498  auto qual_bin_oper = std::dynamic_pointer_cast<Analyzer::BinOper>(join_qual);
499  if (!qual_bin_oper || !IS_EQUIVALENCE(qual_bin_oper->get_optype())) {
500  fail_reasons.emplace_back("No equijoin expression found");
501  if (current_level_join_conditions.type == JoinType::INNER) {
502  add_qualifier_to_execution_unit(ra_exe_unit, join_qual);
503  }
504  continue;
505  }
506  JoinHashTableOrError hash_table_or_error;
507  if (!current_level_hash_table) {
508  hash_table_or_error = buildHashTableForQualifier(
509  qual_bin_oper,
510  query_infos,
514  column_cache);
515  current_level_hash_table = hash_table_or_error.hash_table;
516  }
517  if (hash_table_or_error.hash_table) {
518  plan_state_->join_info_.join_hash_tables_.push_back(hash_table_or_error.hash_table);
519  plan_state_->join_info_.equi_join_tautologies_.push_back(qual_bin_oper);
520  } else {
521  fail_reasons.push_back(hash_table_or_error.fail_reason);
522  if (current_level_join_conditions.type == JoinType::INNER) {
523  add_qualifier_to_execution_unit(ra_exe_unit, qual_bin_oper);
524  }
525  }
526  }
527  return current_level_hash_table;
528 }
529 
530 llvm::Value* Executor::addJoinLoopIterator(const std::vector<llvm::Value*>& prev_iters,
531  const size_t level_idx) {
532  // Iterators are added for loop-outer joins when the head of the loop is generated,
533  // then once again when the body if generated. Allow this instead of special handling
534  // of call sites.
535  const auto it = cgen_state_->scan_idx_to_hash_pos_.find(level_idx);
536  if (it != cgen_state_->scan_idx_to_hash_pos_.end()) {
537  return it->second;
538  }
539  CHECK(!prev_iters.empty());
540  llvm::Value* matching_row_index = prev_iters.back();
541  const auto it_ok =
542  cgen_state_->scan_idx_to_hash_pos_.emplace(level_idx, matching_row_index);
543  CHECK(it_ok.second);
544  return matching_row_index;
545 }
546 
547 void Executor::codegenJoinLoops(const std::vector<JoinLoop>& join_loops,
548  const RelAlgExecutionUnit& ra_exe_unit,
549  GroupByAndAggregate& group_by_and_aggregate,
550  llvm::Function* query_func,
551  llvm::BasicBlock* entry_bb,
553  const CompilationOptions& co,
554  const ExecutionOptions& eo) {
555  const auto exit_bb =
556  llvm::BasicBlock::Create(cgen_state_->context_, "exit", cgen_state_->row_func_);
557  cgen_state_->ir_builder_.SetInsertPoint(exit_bb);
558  cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
559  cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
560  CodeGenerator code_generator(this);
561  const auto loops_entry_bb = JoinLoop::codegen(
562  join_loops,
563  [this,
564  query_func,
565  &query_mem_desc,
566  &co,
567  &eo,
568  &group_by_and_aggregate,
569  &join_loops,
570  &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
571  addJoinLoopIterator(prev_iters, join_loops.size());
572  auto& builder = cgen_state_->ir_builder_;
573  const auto loop_body_bb = llvm::BasicBlock::Create(
574  builder.getContext(), "loop_body", builder.GetInsertBlock()->getParent());
575  builder.SetInsertPoint(loop_body_bb);
576  const bool can_return_error =
577  compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
578  if (can_return_error || cgen_state_->needs_error_check_ ||
580  createErrorCheckControlFlow(query_func,
583  co.device_type);
584  }
585  return loop_body_bb;
586  },
587  code_generator.posArg(nullptr),
588  exit_bb,
590  cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
591  cgen_state_->ir_builder_.CreateBr(loops_entry_bb);
592 }
593 
595  Analyzer::Expr* group_by_col,
596  const size_t col_width,
597  const CompilationOptions& co,
598  const bool translate_null_val,
599  const int64_t translated_null_val,
600  GroupByAndAggregate::DiamondCodegen& diamond_codegen,
601  std::stack<llvm::BasicBlock*>& array_loops,
602  const bool thread_mem_shared) {
603  CHECK_GE(col_width, sizeof(int32_t));
604  CodeGenerator code_generator(this);
605  auto group_key = code_generator.codegen(group_by_col, true, co).front();
606  auto key_to_cache = group_key;
607  if (dynamic_cast<Analyzer::UOper*>(group_by_col) &&
608  static_cast<Analyzer::UOper*>(group_by_col)->get_optype() == kUNNEST) {
609  auto preheader = cgen_state_->ir_builder_.GetInsertBlock();
610  auto array_loop_head = llvm::BasicBlock::Create(cgen_state_->context_,
611  "array_loop_head",
613  preheader->getNextNode());
614  diamond_codegen.setFalseTarget(array_loop_head);
615  const auto ret_ty = get_int_type(32, cgen_state_->context_);
616  auto array_idx_ptr = cgen_state_->ir_builder_.CreateAlloca(ret_ty);
617  CHECK(array_idx_ptr);
618  cgen_state_->ir_builder_.CreateStore(cgen_state_->llInt(int32_t(0)), array_idx_ptr);
619  const auto arr_expr = static_cast<Analyzer::UOper*>(group_by_col)->get_operand();
620  const auto& array_ti = arr_expr->get_type_info();
621  CHECK(array_ti.is_array());
622  const auto& elem_ti = array_ti.get_elem_type();
623  auto array_len =
624  (array_ti.get_size() > 0)
625  ? cgen_state_->llInt(array_ti.get_size() / elem_ti.get_size())
627  "array_size",
628  ret_ty,
629  {group_key,
630  code_generator.posArg(arr_expr),
631  cgen_state_->llInt(log2_bytes(elem_ti.get_logical_size()))});
632  cgen_state_->ir_builder_.CreateBr(array_loop_head);
633  cgen_state_->ir_builder_.SetInsertPoint(array_loop_head);
634  CHECK(array_len);
635  auto array_idx = cgen_state_->ir_builder_.CreateLoad(array_idx_ptr);
636  auto bound_check = cgen_state_->ir_builder_.CreateICmp(
637  llvm::ICmpInst::ICMP_SLT, array_idx, array_len);
638  auto array_loop_body = llvm::BasicBlock::Create(
639  cgen_state_->context_, "array_loop_body", cgen_state_->row_func_);
640  cgen_state_->ir_builder_.CreateCondBr(
641  bound_check,
642  array_loop_body,
643  array_loops.empty() ? diamond_codegen.orig_cond_false_ : array_loops.top());
644  cgen_state_->ir_builder_.SetInsertPoint(array_loop_body);
645  cgen_state_->ir_builder_.CreateStore(
646  cgen_state_->ir_builder_.CreateAdd(array_idx, cgen_state_->llInt(int32_t(1))),
647  array_idx_ptr);
648  auto array_at_fname = "array_at_" + numeric_type_name(elem_ti);
649  if (array_ti.get_size() < 0) {
650  if (array_ti.get_notnull()) {
651  array_at_fname = "notnull_" + array_at_fname;
652  }
653  array_at_fname = "varlen_" + array_at_fname;
654  }
655  const auto ar_ret_ty =
656  elem_ti.is_fp()
657  ? (elem_ti.get_type() == kDOUBLE
658  ? llvm::Type::getDoubleTy(cgen_state_->context_)
659  : llvm::Type::getFloatTy(cgen_state_->context_))
660  : get_int_type(elem_ti.get_logical_size() * 8, cgen_state_->context_);
661  group_key = cgen_state_->emitExternalCall(
662  array_at_fname,
663  ar_ret_ty,
664  {group_key, code_generator.posArg(arr_expr), array_idx});
666  elem_ti, isArchMaxwell(co.device_type), thread_mem_shared)) {
667  key_to_cache = spillDoubleElement(group_key, ar_ret_ty);
668  } else {
669  key_to_cache = group_key;
670  }
671  CHECK(array_loop_head);
672  array_loops.push(array_loop_head);
673  }
674  cgen_state_->group_by_expr_cache_.push_back(key_to_cache);
675  llvm::Value* orig_group_key{nullptr};
676  if (translate_null_val) {
677  const std::string translator_func_name(
678  col_width == sizeof(int32_t) ? "translate_null_key_i32_" : "translate_null_key_");
679  const auto& ti = group_by_col->get_type_info();
680  const auto key_type = get_int_type(ti.get_logical_size() * 8, cgen_state_->context_);
681  orig_group_key = group_key;
682  group_key = cgen_state_->emitCall(
683  translator_func_name + numeric_type_name(ti),
684  {group_key,
685  static_cast<llvm::Value*>(
686  llvm::ConstantInt::get(key_type, inline_int_null_val(ti))),
687  static_cast<llvm::Value*>(llvm::ConstantInt::get(
688  llvm::Type::getInt64Ty(cgen_state_->context_), translated_null_val))});
689  }
690  group_key = cgen_state_->ir_builder_.CreateBitCast(
691  cgen_state_->castToTypeIn(group_key, col_width * 8),
692  get_int_type(col_width * 8, cgen_state_->context_));
693  if (orig_group_key) {
694  orig_group_key = cgen_state_->ir_builder_.CreateBitCast(
695  cgen_state_->castToTypeIn(orig_group_key, col_width * 8),
696  get_int_type(col_width * 8, cgen_state_->context_));
697  }
698  return {group_key, orig_group_key};
699 }
700 
702  Executor* executor,
703  llvm::Value* nullable_lv,
704  const SQLTypeInfo& nullable_ti,
705  const std::string& name)
706  : cgen_state(cgen_state), name(name) {
707  CHECK(nullable_ti.is_number() || nullable_ti.is_decimal() || nullable_ti.is_fp());
708 
709  null_check = std::make_unique<GroupByAndAggregate::DiamondCodegen>(
710  nullable_ti.is_fp()
711  ? cgen_state->ir_builder_.CreateFCmp(llvm::FCmpInst::FCMP_OEQ,
712  nullable_lv,
713  cgen_state->inlineFpNull(nullable_ti))
714  : cgen_state->ir_builder_.CreateICmp(llvm::ICmpInst::ICMP_EQ,
715  nullable_lv,
716  cgen_state->inlineIntNull(nullable_ti)),
717  executor,
718  false,
719  name,
720  nullptr,
721  false);
722 
723  // generate a phi node depending on whether we got a null or not
724  nullcheck_bb =
725  llvm::BasicBlock::Create(cgen_state->context_, name + "_bb", cgen_state->row_func_);
726 
727  // update the blocks created by diamond codegen to point to the newly created phi
728  // block
729  cgen_state->ir_builder_.SetInsertPoint(null_check->cond_true_);
730  cgen_state->ir_builder_.CreateBr(nullcheck_bb);
731  cgen_state->ir_builder_.SetInsertPoint(null_check->cond_false_);
732 }
733 
734 llvm::Value* CodeGenerator::NullCheckCodegen::finalize(llvm::Value* null_lv,
735  llvm::Value* notnull_lv) {
736  CHECK(null_check);
737  cgen_state->ir_builder_.CreateBr(nullcheck_bb);
738 
739  CHECK_EQ(null_lv->getType(), notnull_lv->getType());
740 
741  cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
742  nullcheck_value =
743  cgen_state->ir_builder_.CreatePHI(null_lv->getType(), 2, name + "_value");
744  nullcheck_value->addIncoming(notnull_lv, null_check->cond_false_);
745  nullcheck_value->addIncoming(null_lv, null_check->cond_true_);
746 
747  null_check.reset(nullptr);
748  cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
749  return nullcheck_value;
750 }
JoinInfo join_info_
Definition: PlanState.h:56
catalog_(nullptr)
#define CHECK_EQ(x, y)
Definition: Logger.h:205
llvm::Value * castToTypeIn(llvm::Value *val, const size_t bit_width)
Definition: CgenState.cpp:106
NullCheckCodegen(CgenState *cgen_state, Executor *executor, llvm::Value *nullable_lv, const SQLTypeInfo &nullable_ti, const std::string &name="")
Definition: IRCodegen.cpp:701
#define IS_LOGIC(X)
Definition: sqldefs.h:60
std::vector< llvm::Value * > outer_join_match_found_per_level_
Definition: CgenState.h:330
llvm::BasicBlock * nullcheck_bb
bool is_trivial_loop_join(const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
Definition: Execute.cpp:1115
void codegenJoinLoops(const std::vector< JoinLoop > &join_loops, const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, llvm::Function *query_func, llvm::BasicBlock *entry_bb, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const ExecutionOptions &eo)
Definition: IRCodegen.cpp:547
llvm::Value * values_buffer
Definition: JoinLoop.h:47
#define IS_EQUIVALENCE(X)
Definition: sqldefs.h:67
llvm::Value * codegenArith(const Analyzer::BinOper *, const CompilationOptions &)
CgenState * cgen_state_
bool is_fp() const
Definition: sqltypes.h:419
llvm::Value * emitExternalCall(const std::string &fname, llvm::Type *ret_type, const std::vector< llvm::Value * > args, const std::vector< llvm::Attribute::AttrKind > &fnattrs={}, const bool has_struct_return=false)
Definition: CgenState.h:206
llvm::IRBuilder ir_builder_
Definition: CgenState.h:319
llvm::Value * posArg(const Analyzer::Expr *) const
Definition: ColumnIR.cpp:509
std::string join(T const &container, std::string const &delim)
std::vector< InputDescriptor > input_descs
#define CHECK_GE(x, y)
Definition: Logger.h:210
bool need_patch_unnest_double(const SQLTypeInfo &ti, const bool is_maxwell, const bool mem_shared)
Definition: sqldefs.h:49
llvm::ConstantInt * llBool(const bool v) const
Definition: CgenState.h:310
virtual std::vector< llvm::Value * > codegenColumn(const Analyzer::ColumnVar *, const bool fetch_column, const CompilationOptions &)
Definition: ColumnIR.cpp:93
llvm::Value * codegenArrayAt(const Analyzer::BinOper *, const CompilationOptions &)
Definition: ArrayIR.cpp:25
QualsConjunctiveForm qual_to_conjunctive_form(const std::shared_ptr< Analyzer::Expr > qual_expr)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
bool is_number() const
Definition: sqltypes.h:420
std::vector< llvm::Value * > codegenGeoBinOper(const Analyzer::GeoBinOper *, const CompilationOptions &)
Definition: GeoIR.cpp:80
llvm::Function * row_func_
Definition: CgenState.h:315
llvm::Value * codegenIsNull(const Analyzer::UOper *, const CompilationOptions &)
Definition: LogicalIR.cpp:368
SQLOps get_optype() const
Definition: Analyzer.h:440
false auto cgen_state
std::vector< llvm::Value * > group_by_expr_cache_
Definition: CgenState.h:326
llvm::LLVMContext & context_
Definition: CgenState.h:317
CHECK(cgen_state)
llvm::Value * get_arg_by_name(llvm::Function *func, const std::string &name)
Definition: Execute.h:95
#define INJECT_TIMER(DESC)
Definition: measure.h:91
#define CHECK_NE(x, y)
Definition: Logger.h:206
const bool with_dynamic_watchdog
const JoinQualsPerNestingLevel join_quals
std::vector< llvm::Value * > codegenGeoUOper(const Analyzer::GeoUOper *, const CompilationOptions &)
Definition: GeoIR.cpp:21
llvm::ConstantInt * inlineIntNull(const SQLTypeInfo &)
Definition: CgenState.cpp:27
void setFalseTarget(llvm::BasicBlock *cond_false)
llvm::Value * codegenFunctionOper(const Analyzer::FunctionOper *, const CompilationOptions &)
Executor * executor_
void add_qualifier_to_execution_unit(RelAlgExecutionUnit &ra_exe_unit, const std::shared_ptr< Analyzer::Expr > &qual)
Definition: IRCodegen.cpp:224
std::unordered_map< int, llvm::Value * > scan_idx_to_hash_pos_
Definition: CgenState.h:331
bool needs_error_check_
Definition: CgenState.h:334
std::vector< llvm::Value * > codegenArrayExpr(const Analyzer::ArrayExpr *, const CompilationOptions &)
Definition: ArrayIR.cpp:88
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:79
llvm::Value * codegenUMinus(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
Definition: CgenState.cpp:137
llvm::Value * slot_lookup_result
Definition: JoinLoop.h:45
ExecutorDeviceType device_type
std::unordered_map< int, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
PlanState * plan_state_
std::shared_ptr< JoinHashTableInterface > hash_table
Definition: Execute.h:735
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
Definition: IRCodegen.cpp:26
#define CHECK_LT(x, y)
Definition: Logger.h:207
std::unique_ptr< GroupByAndAggregate::DiamondCodegen > null_check
#define IS_ARITHMETIC(X)
Definition: sqldefs.h:61
const Expr * get_arg() const
Definition: Analyzer.h:751
Expression class for the LOWER (lowercase) string function. The &quot;arg&quot; constructor parameter must be a...
Definition: Analyzer.h:792
llvm::Value * toBool(llvm::Value *)
Definition: LogicalIR.cpp:333
llvm::Value * codegenFunctionOperWithCustomTypeHandling(const Analyzer::FunctionOperWithCustomTypeHandling *, const CompilationOptions &)
llvm::Value * codegenCmp(const Analyzer::BinOper *, const CompilationOptions &)
Definition: CompareIR.cpp:184
static llvm::BasicBlock * codegen(const std::vector< JoinLoop > &join_loops, const std::function< llvm::BasicBlock *(const std::vector< llvm::Value * > &)> &body_codegen, llvm::Value *outer_iter, llvm::BasicBlock *exit_bb, llvm::IRBuilder<> &builder)
Definition: JoinLoop.cpp:45
std::list< std::shared_ptr< Analyzer::Expr > > quals
const bool allow_loop_joins
llvm::ConstantInt * llInt(const T v) const
Definition: CgenState.h:296
llvm::Value * codegenUnnest(const Analyzer::UOper *, const CompilationOptions &)
Definition: ArrayIR.cpp:20
llvm::Value * addJoinLoopIterator(const std::vector< llvm::Value * > &prev_iters, const size_t level_idx)
Definition: IRCodegen.cpp:530
std::list< std::shared_ptr< Analyzer::Expr > > quals
llvm::Value * finalize(llvm::Value *null_lv, llvm::Value *notnull_lv)
Definition: IRCodegen.cpp:734
llvm::Value * codegenLogical(const Analyzer::BinOper *, const CompilationOptions &)
Definition: LogicalIR.cpp:289
void check_if_loop_join_is_allowed(RelAlgExecutionUnit &ra_exe_unit, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, const size_t level_idx, const std::string &fail_reason)
Definition: IRCodegen.cpp:234
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
llvm::ConstantInt * ll_bool(const bool v, llvm::LLVMContext &context)
llvm::Value * codegenCast(const Analyzer::UOper *, const CompilationOptions &)
Definition: CastIR.cpp:20
GroupColLLVMValue groupByColumnCodegen(Analyzer::Expr *group_by_col, const size_t col_width, const CompilationOptions &, const bool translate_null_val, const int64_t translated_null_val, GroupByAndAggregate::DiamondCodegen &, std::stack< llvm::BasicBlock * > &, const bool thread_mem_shared)
Definition: IRCodegen.cpp:594
uint32_t log2_bytes(const uint32_t bytes)
Definition: Execute.h:105
std::string numeric_type_name(const SQLTypeInfo &ti)
Definition: Execute.h:138
std::vector< JoinLoop > buildJoinLoops(RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache)
Definition: IRCodegen.cpp:256
std::function< llvm::Value *(const std::vector< llvm::Value * > &, llvm::Value *)> buildIsDeletedCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const CompilationOptions &co)
Definition: IRCodegen.cpp:425
const bool allow_runtime_query_interrupt
bool is_decimal() const
Definition: sqltypes.h:418
Definition: sqldefs.h:39
SQLOps get_optype() const
Definition: Analyzer.h:371
#define VLOG(n)
Definition: Logger.h:291
std::shared_ptr< JoinHashTableInterface > buildCurrentLevelHashTable(const JoinCondition &current_level_join_conditions, RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache, std::vector< std::string > &fail_reasons)
Definition: IRCodegen.cpp:484
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals
#define IS_COMPARISON(X)
Definition: sqldefs.h:57
llvm::ConstantFP * inlineFpNull(const SQLTypeInfo &)
Definition: CgenState.cpp:65
Executor * executor() const