17 #include "../Parser/ParserNode.h"
31 const bool fetch_columns,
57 if (ti.get_type() ==
kNULLT) {
58 throw std::runtime_error(
59 "NULL type literals are not currently supported in this context.");
61 if (constant->get_is_null()) {
63 ?
static_cast<llvm::Value*
>(
executor_->cgen_state_->inlineFpNull(ti))
64 : static_cast<llvm::Value*>(
executor_->cgen_state_->inlineIntNull(ti))};
70 return {
codegen(constant, ti.get_compression(), ti.get_comp_param(), co)};
72 return {
codegen(constant, ti.get_compression(), 0, co)};
76 return {
codegen(case_expr, co)};
80 return {
codegen(extract_expr, co)};
84 return {
codegen(dateadd_expr, co)};
88 return {
codegen(datediff_expr, co)};
92 return {
codegen(datetrunc_expr, co)};
95 if (charlength_expr) {
96 return {
codegen(charlength_expr, co)};
99 if (keyforstring_expr) {
100 return {
codegen(keyforstring_expr, co)};
103 if (sample_ratio_expr) {
104 return {
codegen(sample_ratio_expr, co)};
107 if (string_oper_expr) {
108 return {
codegen(string_oper_expr, co)};
111 if (cardinality_expr) {
112 return {
codegen(cardinality_expr, co)};
116 return {
codegen(like_expr, co)};
120 return {
codegen(regexp_expr, co)};
127 if (likelihood_expr) {
128 return {
codegen(likelihood_expr->get_arg(), fetch_columns, co)};
135 if (in_integer_set_expr) {
136 return {
codegen(in_integer_set_expr, co)};
138 auto function_oper_with_custom_type_handling_expr =
140 if (function_oper_with_custom_type_handling_expr) {
142 function_oper_with_custom_type_handling_expr, co)};
145 if (array_oper_expr) {
157 if (function_oper_expr) {
164 if (dynamic_cast<const Analyzer::OffsetInFragment*>(expr)) {
167 if (dynamic_cast<const Analyzer::WindowFunction*>(expr)) {
220 auto input_expr = expr->
get_arg();
223 auto double_lv =
codegen(input_expr,
true, co);
224 CHECK_EQ(
size_t(1), double_lv.size());
226 std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
227 const bool is_nullable = !input_expr->get_type_info().get_notnull();
229 nullcheck_codegen = std::make_unique<NullCheckCodegen>(
cgen_state_,
232 input_expr->get_type_info(),
233 "sample_ratio_nullcheck");
236 std::vector<llvm::Value*>
args{double_lv[0],
posArg(
nullptr)};
238 if (nullcheck_codegen) {
251 CHECK(target_value_expr);
252 CHECK(lower_bound_expr);
253 CHECK(upper_bound_expr);
254 CHECK(partition_count_expr);
257 auto target_expr = expr;
258 if (
auto cast_expr = dynamic_cast<const Analyzer::UOper*>(expr)) {
260 target_expr = cast_expr->get_operand();
272 if (is_constant_expr(lower_bound_expr) && is_constant_expr(upper_bound_expr) &&
273 is_constant_expr(partition_count_expr)) {
287 if (!col_range.hasNulls()) {
312 if (num_partitions < 1 || num_partitions > INT32_MAX) {
313 throw std::runtime_error(
314 "PARTITION_COUNT expression of width_bucket function should be in a valid "
315 "range: 0 < PARTITION_COUNT <= 2147483647");
319 if (lower == upper) {
320 throw std::runtime_error(
321 "LOWER_BOUND and UPPER_BOUND expressions of width_bucket function cannot have "
322 "the same constant value");
325 throw std::runtime_error(
326 "Both LOWER_BOUND and UPPER_BOUND of width_bucket function should be finite "
327 "numeric constants.");
330 bool const reversed = lower > upper;
331 double const scale_factor = num_partitions / (reversed ? lower - upper : upper - lower);
332 std::string func_name = reversed ?
"width_bucket_reversed" :
"width_bucket";
334 auto get_double_constant_lvs = [
this, &co](
double const_val) {
337 auto double_const_expr =
339 return codegen(double_const_expr.get(),
false, co);
342 auto target_value_ti = target_value_expr->get_type_info();
343 auto target_value_expr_lvs =
codegen(target_value_expr,
true, co);
344 CHECK_EQ(
size_t(1), target_value_expr_lvs.size());
345 auto lower_expr_lvs =
codegen(lower_bound_expr,
true, co);
346 CHECK_EQ(
size_t(1), lower_expr_lvs.size());
347 auto scale_factor_lvs = get_double_constant_lvs(scale_factor);
348 CHECK_EQ(
size_t(1), scale_factor_lvs.size());
350 std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
353 func_name +=
"_no_oob_check";
354 width_bucket_args.push_back(scale_factor_lvs[0]);
356 auto upper_expr_lvs =
codegen(upper_bound_expr,
true, co);
357 CHECK_EQ(
size_t(1), upper_expr_lvs.size());
358 auto partition_count_expr_lvs =
codegen(partition_count_expr,
true, co);
359 CHECK_EQ(
size_t(1), partition_count_expr_lvs.size());
360 width_bucket_args.push_back(upper_expr_lvs[0]);
361 width_bucket_args.push_back(scale_factor_lvs[0]);
362 width_bucket_args.push_back(partition_count_expr_lvs[0]);
363 if (!target_value_ti.get_notnull()) {
364 func_name +=
"_nullable";
365 auto translated_null_value = target_value_ti.is_fp()
368 auto null_value_lvs = get_double_constant_lvs(translated_null_value);
369 CHECK_EQ(
size_t(1), null_value_lvs.size());
370 width_bucket_args.push_back(null_value_lvs[0]);
383 std::string func_name =
"width_bucket_expr";
384 bool nullable_expr =
false;
386 func_name +=
"_no_oob_check";
387 }
else if (!target_value_expr->get_type_info().get_notnull()) {
388 func_name +=
"_nullable";
389 nullable_expr =
true;
392 auto target_value_expr_lvs =
codegen(target_value_expr,
true, co);
393 CHECK_EQ(
size_t(1), target_value_expr_lvs.size());
394 auto lower_bound_expr_lvs =
codegen(lower_bound_expr,
true, co);
395 CHECK_EQ(
size_t(1), lower_bound_expr_lvs.size());
396 auto upper_bound_expr_lvs =
codegen(upper_bound_expr,
true, co);
397 CHECK_EQ(
size_t(1), upper_bound_expr_lvs.size());
398 auto partition_count_expr_lvs =
codegen(partition_count_expr,
true, co);
399 CHECK_EQ(
size_t(1), partition_count_expr_lvs.size());
400 auto target_value_ti = target_value_expr->get_type_info();
405 auto partition_count_ti = partition_count_expr->get_type_info();
406 CHECK(partition_count_ti.is_integer());
408 auto partition_count_expr_lv =
412 partition_count_ti.get_size() < int32_ti.get_size());
414 llvm::Value* partition_count_min =
416 llvm::BasicBlock* width_bucket_partition_count_ok_bb =
418 "width_bucket_partition_count_ok_bb",
420 llvm::BasicBlock* width_bucket_argument_check_fail_bb =
422 "width_bucket_argument_check_fail_bb",
425 width_bucket_argument_check_fail_bb,
426 width_bucket_partition_count_ok_bb);
432 llvm::BasicBlock* width_bucket_bound_check_ok_bb =
434 "width_bucket_bound_check_ok_bb",
436 llvm::Value* bound_check{
nullptr};
437 if (lower_bound_expr->get_type_info().get_notnull() &&
438 upper_bound_expr->get_type_info().get_notnull()) {
440 lower_bound_expr_lvs[0], upper_bound_expr_lvs[0],
"bound_check");
442 std::vector<llvm::Value*> bound_check_args{
443 lower_bound_expr_lvs[0],
444 upper_bound_expr_lvs[0],
450 bound_check, width_bucket_argument_check_fail_bb, width_bucket_bound_check_ok_bb);
455 lower_bound_expr_lvs,
456 lower_bound_expr->get_type_info(),
459 auto lower_bound_expr_lv = lower_bound_expr_lvs[0];
460 auto upper_bound_expr_lv = upper_bound_expr_lvs[0];
461 std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
465 partition_count_expr_lv};
467 width_bucket_args.push_back(null_value_lv);
475 const std::shared_ptr<Analyzer::Expr>& qual) {
478 qual_cf.simple_quals.begin(),
479 qual_cf.simple_quals.end());
480 ra_exe_unit.
quals.insert(
481 ra_exe_unit.
quals.end(), qual_cf.quals.begin(), qual_cf.quals.end());
486 const std::vector<InputTableInfo>& query_infos,
487 const size_t level_idx,
488 const std::string& fail_reason) {
492 if (level_idx + 1 != ra_exe_unit.
join_quals.size()) {
493 throw std::runtime_error(
494 "Hash join failed, reason(s): " + fail_reason +
495 " | Cannot fall back to loop join for intermediate join quals");
498 throw std::runtime_error(
499 "Hash join failed, reason(s): " + fail_reason +
500 " | Cannot fall back to loop join for non-trivial inner table size");
509 if (lhs_cv && rhs_cv && !bin_oper->is_overlaps_oper()) {
511 auto rhs_type = rhs_cv->get_type_info().get_type();
514 throw std::runtime_error(
515 "Join operation between full array columns (i.e., R.arr = S.arr) instead of "
516 "indexed array columns (i.e., R.arr[1] = S.arr[2]) is not supported yet.");
527 const std::vector<InputTableInfo>& query_infos,
531 std::vector<JoinLoop> join_loops;
532 for (
size_t level_idx = 0, current_hash_table_idx = 0;
535 const auto& current_level_join_conditions = ra_exe_unit.
join_quals[level_idx];
536 std::vector<std::string> fail_reasons;
537 const auto current_level_hash_table =
538 buildCurrentLevelHashTable(current_level_join_conditions,
545 const auto found_outer_join_matches_cb =
546 [
this, level_idx](llvm::Value* found_outer_join_matches) {
550 found_outer_join_matches;
552 const auto is_deleted_cb = buildIsDeletedCb(ra_exe_unit, level_idx, co);
553 auto rem_left_join_quals_it =
555 bool has_remaining_left_join_quals =
557 !rem_left_join_quals_it->second.empty();
558 const auto outer_join_condition_remaining_quals_cb =
559 [
this, level_idx, &co](
const std::vector<llvm::Value*>& prev_iters) {
564 addJoinLoopIterator(prev_iters, level_idx + 1);
569 for (
auto expr : it->second) {
573 code_generator.
codegen(expr.get(),
true, co).front()));
576 return left_join_cond;
578 if (current_level_hash_table) {
579 const auto hoisted_filters_cb = buildHoistLeftHandSideFiltersCb(
580 ra_exe_unit, level_idx, current_level_hash_table->getInnerTableId(), co);
582 join_loops.emplace_back(
584 current_level_join_conditions.type,
586 [
this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
587 const std::vector<llvm::Value*>& prev_iters) {
588 addJoinLoopIterator(prev_iters, level_idx);
591 current_level_hash_table->codegenSlot(co, current_hash_table_idx);
596 has_remaining_left_join_quals
597 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
598 outer_join_condition_remaining_quals_cb)
601 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
606 }
else if (
auto range_join_table =
607 dynamic_cast<RangeJoinHashTable*>(current_level_hash_table.get())) {
608 join_loops.emplace_back(
610 current_level_join_conditions.type,
614 current_hash_table_idx,
616 current_level_hash_table,
617 &co](
const std::vector<llvm::Value*>& prev_iters) {
618 addJoinLoopIterator(prev_iters, level_idx);
620 CHECK(!prev_iters.empty());
621 const auto matching_set = range_join_table->codegenMatchingSetWithOffset(
622 co, current_hash_table_idx, prev_iters.back());
623 domain.values_buffer = matching_set.elements;
624 domain.element_count = matching_set.count;
629 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
630 outer_join_condition_remaining_quals_cb)
634 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
640 join_loops.emplace_back(
642 current_level_join_conditions.type,
644 [
this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
645 const std::vector<llvm::Value*>& prev_iters) {
646 addJoinLoopIterator(prev_iters, level_idx);
648 const auto matching_set = current_level_hash_table->codegenMatchingSet(
649 co, current_hash_table_idx);
651 domain.element_count = matching_set.count;
656 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
657 outer_join_condition_remaining_quals_cb)
660 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
666 ++current_hash_table_idx;
668 const auto fail_reasons_str = current_level_join_conditions.quals.empty()
669 ?
"No equijoin expression found"
672 ra_exe_unit, eo, query_infos, level_idx, fail_reasons_str);
675 VLOG(1) <<
"Unable to build hash table, falling back to loop join: "
677 const auto outer_join_condition_cb =
678 [
this, level_idx, &co, ¤t_level_join_conditions](
679 const std::vector<llvm::Value*>& prev_iters) {
684 addJoinLoopIterator(prev_iters, level_idx + 1);
687 for (
auto expr : current_level_join_conditions.quals) {
690 code_generator.toBool(
691 code_generator.codegen(expr.get(),
true, co).front()));
693 return left_join_cond;
695 join_loops.emplace_back(
697 current_level_join_conditions.type,
699 [
this, level_idx](
const std::vector<llvm::Value*>& prev_iters) {
700 addJoinLoopIterator(prev_iters, level_idx);
704 arg->getType()->getScalarType()->getPointerElementType(),
708 rows_per_scan_ptr->getType()->getPointerElementType(),
710 "num_rows_per_scan");
715 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
716 outer_join_condition_cb)
720 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
735 return {col_expr->get_table_id()};
740 for (
size_t i = 0; i < func_expr->getArity(); i++) {
741 ret = aggregateResult(ret, visit(func_expr->getArg(i)));
748 ret = aggregateResult(ret, visit(bin_oper->get_left_operand()));
749 return aggregateResult(ret, visit(bin_oper->get_right_operand()));
753 return visit(u_oper->get_operand());
757 const std::set<int>& next_result)
const final {
758 auto ret = aggregate;
759 for (
const auto& el : next_result) {
770 const size_t level_idx,
771 const int inner_table_id,
777 const auto& current_level_join_conditions = ra_exe_unit.
join_quals[level_idx];
778 if (level_idx == 0 && current_level_join_conditions.type ==
JoinType::LEFT) {
779 const auto& condition = current_level_join_conditions.quals.front();
781 CHECK(bin_oper) << condition->toString();
786 if (lhs && rhs && lhs->get_table_id() != rhs->get_table_id()) {
791 if (lhs->get_table_id() == inner_table_id) {
793 }
else if (rhs->get_table_id() == inner_table_id) {
797 std::list<std::shared_ptr<Analyzer::Expr>> hoisted_quals;
799 auto should_hoist_qual = [&hoisted_quals](
const auto& qual,
const int table_id) {
802 ExprTableIdVisitor visitor;
803 const auto table_ids = visitor.visit(qual.get());
804 if (table_ids.size() == 1 && table_ids.find(table_id) != table_ids.end()) {
805 hoisted_quals.push_back(qual);
809 should_hoist_qual(qual, selected_lhs->get_table_id());
811 for (
const auto& qual : ra_exe_unit.
quals) {
812 should_hoist_qual(qual, selected_lhs->get_table_id());
816 if (!hoisted_quals.empty()) {
817 return [
this, hoisted_quals, co](llvm::BasicBlock* true_bb,
818 llvm::BasicBlock* exit_bb,
819 const std::string& loop_name,
820 llvm::Function* parent_func,
821 CgenState* cgen_state) -> llvm::BasicBlock* {
823 bool has_quals_to_hoist =
false;
824 for (
const auto& qual : hoisted_quals) {
827 if (plan_state_->hoisted_filters_.count(qual) == 0) {
828 has_quals_to_hoist =
true;
833 if (!has_quals_to_hoist) {
839 llvm::IRBuilder<>& builder = cgen_state->ir_builder_;
840 auto& context = builder.getContext();
842 const auto filter_bb =
843 llvm::BasicBlock::Create(context,
844 "hoisted_left_join_filters_" + loop_name,
847 builder.SetInsertPoint(filter_bb);
849 llvm::Value* filter_lv = cgen_state_->llBool(
true);
852 for (
const auto& qual : hoisted_quals) {
853 if (plan_state_->hoisted_filters_.insert(qual).second) {
856 VLOG(1) <<
"Generating code for hoisted left hand side qualifier "
858 auto cond = code_generator.
toBool(
859 code_generator.
codegen(qual.get(),
true, co).front());
860 filter_lv = builder.CreateAnd(filter_lv, cond);
863 CHECK(filter_lv->getType()->isIntegerTy(1));
865 builder.CreateCondBr(filter_lv, true_bb, exit_bb);
875 std::function<llvm::Value*(const std::vector<llvm::Value*>&, llvm::Value*)>
877 const size_t level_idx,
884 const auto input_desc = ra_exe_unit.
input_descs[level_idx + 1];
889 const auto deleted_cd = plan_state_->getDeletedColForTable(input_desc.getTableId());
893 CHECK(deleted_cd->columnType.is_boolean());
894 const auto deleted_expr = makeExpr<Analyzer::ColumnVar>(deleted_cd->columnType,
895 input_desc.getTableId(),
896 deleted_cd->columnId,
897 input_desc.getNestLevel());
898 return [
this, deleted_expr, level_idx, &co](
const std::vector<llvm::Value*>& prev_iters,
899 llvm::Value* have_more_inner_rows) {
900 const auto matching_row_index = addJoinLoopIterator(prev_iters, level_idx + 1);
904 llvm::Value* is_valid_it{
nullptr};
905 if (have_more_inner_rows) {
906 is_valid_it = have_more_inner_rows;
908 is_valid_it = cgen_state_->ir_builder_.CreateICmp(
909 llvm::ICmpInst::ICMP_SGE, matching_row_index, cgen_state_->llInt<int64_t>(0));
911 const auto it_valid_bb = llvm::BasicBlock::Create(
912 cgen_state_->context_,
"it_valid", cgen_state_->current_func_);
913 const auto it_not_valid_bb = llvm::BasicBlock::Create(
914 cgen_state_->context_,
"it_not_valid", cgen_state_->current_func_);
915 cgen_state_->ir_builder_.CreateCondBr(is_valid_it, it_valid_bb, it_not_valid_bb);
916 const auto row_is_deleted_bb = llvm::BasicBlock::Create(
917 cgen_state_->context_,
"row_is_deleted", cgen_state_->current_func_);
918 cgen_state_->ir_builder_.SetInsertPoint(it_valid_bb);
920 const auto row_is_deleted = code_generator.
toBool(
921 code_generator.
codegen(deleted_expr.get(),
true, co).front());
922 cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
923 cgen_state_->ir_builder_.SetInsertPoint(it_not_valid_bb);
924 const auto row_is_deleted_default = cgen_state_->llBool(
false);
925 cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
926 cgen_state_->ir_builder_.SetInsertPoint(row_is_deleted_bb);
927 auto row_is_deleted_or_default =
928 cgen_state_->ir_builder_.CreatePHI(row_is_deleted->getType(), 2);
929 row_is_deleted_or_default->addIncoming(row_is_deleted, it_valid_bb);
930 row_is_deleted_or_default->addIncoming(row_is_deleted_default, it_not_valid_bb);
931 return row_is_deleted_or_default;
940 const std::vector<InputTableInfo>& query_infos,
942 std::vector<std::string>& fail_reasons) {
944 std::shared_ptr<HashJoin> current_level_hash_table;
945 auto handleNonHashtableQual = [&ra_exe_unit, &level_idx,
this](
947 std::shared_ptr<Analyzer::Expr> qual) {
949 plan_state_->addNonHashtableQualForLeftJoin(level_idx, qual);
954 for (
const auto& join_qual : current_level_join_conditions.
quals) {
956 if (current_level_hash_table || !qual_bin_oper ||
958 handleNonHashtableQual(current_level_join_conditions.
type, join_qual);
959 if (!current_level_hash_table) {
960 fail_reasons.emplace_back(
"No equijoin expression found");
966 if (!current_level_hash_table) {
967 hash_table_or_error = buildHashTableForQualifier(
972 current_level_join_conditions.
type,
978 current_level_hash_table = hash_table_or_error.
hash_table;
981 plan_state_->join_info_.join_hash_tables_.push_back(hash_table_or_error.
hash_table);
982 plan_state_->join_info_.equi_join_tautologies_.push_back(qual_bin_oper);
984 fail_reasons.push_back(hash_table_or_error.
fail_reason);
985 if (!current_level_hash_table) {
986 VLOG(2) <<
"Building a hashtable based on a qual " << qual_bin_oper->toString()
989 handleNonHashtableQual(current_level_join_conditions.
type, qual_bin_oper);
992 return current_level_hash_table;
996 if (!cgen_state_->filter_func_) {
1004 for (
auto bb_it = cgen_state_->filter_func_->begin();
1005 bb_it != cgen_state_->filter_func_->end();
1007 for (
auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1009 for (
auto op_it = instr_it->value_op_begin(); op_it != instr_it->value_op_end();
1011 llvm::Value* v = *op_it;
1015 if (llvm::dyn_cast<const llvm::CallInst>(instr_it) &&
1016 op_it == instr_it->value_op_end() - 1) {
1021 if (
auto* instr = llvm::dyn_cast<llvm::Instruction>(v);
1022 instr && instr->getParent() &&
1023 instr->getParent()->getParent() == cgen_state_->row_func_) {
1025 cgen_state_->filter_func_args_[v] =
nullptr;
1026 }
else if (
auto* argum = llvm::dyn_cast<llvm::Argument>(v);
1027 argum && argum->getParent() == cgen_state_->row_func_) {
1029 cgen_state_->filter_func_args_[v] =
nullptr;
1037 std::vector<llvm::Type*> filter_func_arg_types;
1038 filter_func_arg_types.reserve(cgen_state_->filter_func_args_.v_.size());
1039 for (
auto& arg : cgen_state_->filter_func_args_.v_) {
1040 filter_func_arg_types.push_back(arg->getType());
1042 auto ft = llvm::FunctionType::get(
1043 get_int_type(32, cgen_state_->context_), filter_func_arg_types,
false);
1044 cgen_state_->filter_func_->setName(
"old_filter_func");
1045 auto filter_func2 = llvm::Function::Create(ft,
1046 llvm::Function::ExternalLinkage,
1048 cgen_state_->filter_func_->getParent());
1049 CHECK_EQ(filter_func2->arg_size(), cgen_state_->filter_func_args_.v_.size());
1050 auto arg_it = cgen_state_->filter_func_args_.begin();
1052 for (llvm::Function::arg_iterator I = filter_func2->arg_begin(),
1053 E = filter_func2->arg_end();
1056 arg_it->second = &*I;
1057 if (arg_it->first->hasName()) {
1058 I->setName(arg_it->first->getName());
1067 filter_func2->getBasicBlockList().splice(
1068 filter_func2->begin(), cgen_state_->filter_func_->getBasicBlockList());
1070 if (cgen_state_->current_func_ == cgen_state_->filter_func_) {
1071 cgen_state_->current_func_ = filter_func2;
1073 cgen_state_->filter_func_ = filter_func2;
1076 for (
auto bb_it = cgen_state_->filter_func_->begin();
1077 bb_it != cgen_state_->filter_func_->end();
1079 for (
auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1081 for (
auto op_it = instr_it->op_begin(); op_it != instr_it->op_end(); ++op_it, ++i) {
1082 llvm::Value* v = op_it->get();
1083 if (
auto arg_it = cgen_state_->filter_func_args_.find(v);
1084 arg_it != cgen_state_->filter_func_args_.end()) {
1086 llvm::Use* use = &*op_it;
1087 use->set(arg_it->second);
1095 const size_t level_idx) {
1100 const auto it = cgen_state_->scan_idx_to_hash_pos_.find(level_idx);
1101 if (it != cgen_state_->scan_idx_to_hash_pos_.end()) {
1104 CHECK(!prev_iters.empty());
1105 llvm::Value* matching_row_index = prev_iters.back();
1107 cgen_state_->scan_idx_to_hash_pos_.emplace(level_idx, matching_row_index);
1108 CHECK(it_ok.second);
1109 return matching_row_index;
1115 llvm::Function* query_func,
1116 llvm::BasicBlock* entry_bb,
1121 const auto exit_bb =
1122 llvm::BasicBlock::Create(cgen_state_->context_,
"exit", cgen_state_->current_func_);
1123 cgen_state_->ir_builder_.SetInsertPoint(exit_bb);
1124 cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
1125 cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1128 llvm::BasicBlock* loops_entry_bb{
nullptr};
1129 auto has_range_join =
1130 std::any_of(join_loops.begin(), join_loops.end(), [](
const auto& join_loop) {
1133 if (has_range_join) {
1134 CHECK_EQ(join_loops.size(), size_t(1));
1135 const auto element_count =
1136 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_), 9);
1138 auto compute_packed_offset = [](
const int32_t x,
const int32_t y) -> uint64_t {
1139 const uint64_t y_shifted =
static_cast<uint64_t
>(y) << 32;
1140 return y_shifted |
static_cast<uint32_t
>(x);
1143 const auto values_arr = std::vector<llvm::Constant*>{
1144 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_), 0),
1145 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1146 compute_packed_offset(0, 1)),
1147 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1148 compute_packed_offset(0, -1)),
1149 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1150 compute_packed_offset(1, 0)),
1151 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1152 compute_packed_offset(1, 1)),
1153 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1154 compute_packed_offset(1, -1)),
1155 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1156 compute_packed_offset(-1, 0)),
1157 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1158 compute_packed_offset(-1, 1)),
1159 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1160 compute_packed_offset(-1, -1))};
1162 const auto constant_values_array = llvm::ConstantArray::get(
1164 CHECK(cgen_state_->module_);
1166 new llvm::GlobalVariable(*cgen_state_->module_,
1169 llvm::GlobalValue::LinkageTypes::InternalLinkage,
1170 constant_values_array);
1174 [element_count, values](
const std::vector<llvm::Value*>& v) {
1177 domain.values_buffer = values;
1193 &group_by_and_aggregate,
1195 &ra_exe_unit](
const std::vector<llvm::Value*>& prev_iters) {
1196 auto& builder = cgen_state_->ir_builder_;
1199 llvm::BasicBlock::Create(cgen_state_->context_,
1200 "range_key_inner_body_exit",
1201 builder.GetInsertBlock()->getParent());
1203 auto range_key_body_bb =
1204 llvm::BasicBlock::Create(cgen_state_->context_,
1205 "range_key_loop_body",
1206 builder.GetInsertBlock()->getParent());
1207 builder.SetInsertPoint(range_key_body_bb);
1216 &group_by_and_aggregate,
1218 &ra_exe_unit](
const std::vector<llvm::Value*>& prev_iters) {
1219 addJoinLoopIterator(prev_iters, join_loops.size());
1220 auto& builder = cgen_state_->ir_builder_;
1221 const auto loop_body_bb =
1222 llvm::BasicBlock::Create(builder.getContext(),
1224 builder.GetInsertBlock()->getParent());
1225 builder.SetInsertPoint(loop_body_bb);
1226 const bool can_return_error =
1227 compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1228 if (can_return_error || cgen_state_->needs_error_check_ ||
1229 eo.with_dynamic_watchdog || eo.allow_runtime_query_interrupt) {
1230 createErrorCheckControlFlow(query_func,
1231 eo.with_dynamic_watchdog,
1232 eo.allow_runtime_query_interrupt,
1235 group_by_and_aggregate.query_infos_);
1237 return loop_body_bb;
1243 builder.SetInsertPoint(range_key_body_bb);
1244 cgen_state_->ir_builder_.CreateBr(body_loops_entry_bb);
1246 builder.SetInsertPoint(body_exit_bb);
1247 return range_key_body_bb;
1249 code_generator.
posArg(
nullptr),
1261 &group_by_and_aggregate,
1263 &ra_exe_unit](
const std::vector<llvm::Value*>& prev_iters) {
1265 addJoinLoopIterator(prev_iters, join_loops.size());
1266 auto& builder = cgen_state_->ir_builder_;
1267 const auto loop_body_bb = llvm::BasicBlock::Create(
1268 builder.getContext(),
"loop_body", builder.GetInsertBlock()->getParent());
1269 builder.SetInsertPoint(loop_body_bb);
1270 const bool can_return_error =
1271 compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1272 if (can_return_error || cgen_state_->needs_error_check_ ||
1274 createErrorCheckControlFlow(query_func,
1281 return loop_body_bb;
1283 code_generator.
posArg(
nullptr),
1287 CHECK(loops_entry_bb);
1288 cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1289 cgen_state_->ir_builder_.CreateBr(loops_entry_bb);
1294 const size_t col_width,
1296 const bool translate_null_val,
1297 const int64_t translated_null_val,
1299 std::stack<llvm::BasicBlock*>& array_loops,
1300 const bool thread_mem_shared) {
1302 CHECK_GE(col_width,
sizeof(int32_t));
1304 auto group_key = code_generator.
codegen(group_by_col,
true, co).front();
1305 auto key_to_cache = group_key;
1306 if (dynamic_cast<Analyzer::UOper*>(group_by_col) &&
1307 static_cast<Analyzer::UOper*>(group_by_col)->get_optype() ==
kUNNEST) {
1308 auto preheader = cgen_state_->ir_builder_.GetInsertBlock();
1309 auto array_loop_head = llvm::BasicBlock::Create(cgen_state_->context_,
1311 cgen_state_->current_func_,
1312 preheader->getNextNode());
1314 const auto ret_ty =
get_int_type(32, cgen_state_->context_);
1315 auto array_idx_ptr = cgen_state_->ir_builder_.CreateAlloca(ret_ty);
1316 CHECK(array_idx_ptr);
1317 cgen_state_->ir_builder_.CreateStore(cgen_state_->llInt(int32_t(0)), array_idx_ptr);
1318 const auto arr_expr =
static_cast<Analyzer::UOper*
>(group_by_col)->get_operand();
1320 CHECK(array_ti.is_array());
1321 const auto& elem_ti = array_ti.get_elem_type();
1323 (array_ti.get_size() > 0)
1324 ? cgen_state_->llInt(array_ti.get_size() / elem_ti.get_size())
1325 : cgen_state_->emitExternalCall(
1329 code_generator.
posArg(arr_expr),
1330 cgen_state_->llInt(
log2_bytes(elem_ti.get_logical_size()))});
1331 cgen_state_->ir_builder_.CreateBr(array_loop_head);
1332 cgen_state_->ir_builder_.SetInsertPoint(array_loop_head);
1334 auto array_idx = cgen_state_->ir_builder_.CreateLoad(
1335 array_idx_ptr->getType()->getPointerElementType(), array_idx_ptr);
1336 auto bound_check = cgen_state_->ir_builder_.CreateICmp(
1337 llvm::ICmpInst::ICMP_SLT, array_idx, array_len);
1338 auto array_loop_body = llvm::BasicBlock::Create(
1339 cgen_state_->context_,
"array_loop_body", cgen_state_->current_func_);
1340 cgen_state_->ir_builder_.CreateCondBr(
1343 array_loops.empty() ? diamond_codegen.
orig_cond_false_ : array_loops.top());
1344 cgen_state_->ir_builder_.SetInsertPoint(array_loop_body);
1345 cgen_state_->ir_builder_.CreateStore(
1346 cgen_state_->ir_builder_.CreateAdd(array_idx, cgen_state_->llInt(int32_t(1))),
1349 if (array_ti.get_size() < 0) {
1350 if (array_ti.get_notnull()) {
1351 array_at_fname =
"notnull_" + array_at_fname;
1353 array_at_fname =
"varlen_" + array_at_fname;
1355 const auto ar_ret_ty =
1357 ? (elem_ti.get_type() ==
kDOUBLE
1358 ? llvm::Type::getDoubleTy(cgen_state_->context_)
1359 : llvm::Type::getFloatTy(cgen_state_->context_))
1360 :
get_int_type(elem_ti.get_logical_size() * 8, cgen_state_->context_);
1361 group_key = cgen_state_->emitExternalCall(
1364 {group_key, code_generator.
posArg(arr_expr), array_idx});
1366 elem_ti, isArchMaxwell(co.
device_type), thread_mem_shared)) {
1367 key_to_cache = spillDoubleElement(group_key, ar_ret_ty);
1369 key_to_cache = group_key;
1371 CHECK(array_loop_head);
1372 array_loops.push(array_loop_head);
1374 cgen_state_->group_by_expr_cache_.push_back(key_to_cache);
1375 llvm::Value* orig_group_key{
nullptr};
1376 if (translate_null_val) {
1377 const std::string translator_func_name(
1378 col_width ==
sizeof(int32_t) ?
"translate_null_key_i32_" :
"translate_null_key_");
1379 const auto& ti = group_by_col->get_type_info();
1380 const auto key_type =
get_int_type(ti.get_logical_size() * 8, cgen_state_->context_);
1381 orig_group_key = group_key;
1382 group_key = cgen_state_->emitCall(
1385 static_cast<llvm::Value*
>(
1387 static_cast<llvm::Value*>(llvm::ConstantInt::get(
1388 llvm::Type::getInt64Ty(cgen_state_->context_), translated_null_val))});
1390 group_key = cgen_state_->ir_builder_.CreateBitCast(
1391 cgen_state_->castToTypeIn(group_key, col_width * 8),
1393 if (orig_group_key) {
1394 orig_group_key = cgen_state_->ir_builder_.CreateBitCast(
1395 cgen_state_->castToTypeIn(orig_group_key, col_width * 8),
1398 return {group_key, orig_group_key};
1403 llvm::Value* nullable_lv,
1405 const std::string&
name)
1406 : cgen_state(cgen_state), name(name) {
1411 llvm::Value* is_null_lv{
nullptr};
1412 if (nullable_ti.
is_fp()) {
1414 llvm::FCmpInst::FCMP_OEQ, nullable_lv, cgen_state->
inlineFpNull(nullable_ti));
1417 llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->
llBool(
true));
1420 llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->
inlineIntNull(nullable_ti));
1424 std::make_unique<DiamondCodegen>(is_null_lv,
executor,
false,
name,
nullptr,
false);
1438 llvm::Value* notnull_lv) {
1441 cgen_state->ir_builder_.CreateBr(nullcheck_bb);
1443 CHECK_EQ(null_lv->getType(), notnull_lv->getType());
1445 cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1447 cgen_state->ir_builder_.CreatePHI(null_lv->getType(), 2,
name +
"_value");
1448 nullcheck_value->addIncoming(notnull_lv, null_check->cond_false_);
1449 nullcheck_value->addIncoming(null_lv, null_check->cond_true_);
1451 null_check.reset(
nullptr);
1452 cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1453 return nullcheck_value;
bool g_enable_left_join_filter_hoisting
NullCheckCodegen(CgenState *cgen_state, Executor *executor, llvm::Value *nullable_lv, const SQLTypeInfo &nullable_ti, const std::string &name="")
void codegenJoinLoops(const std::vector< JoinLoop > &join_loops, const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, llvm::Function *query_func, llvm::BasicBlock *entry_bb, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const ExecutionOptions &eo)
const Expr * get_partition_count() const
std::vector< llvm::Value * > outer_join_match_found_per_level_
bool is_constant_expr() const
std::unordered_map< size_t, std::vector< std::shared_ptr< Analyzer::Expr > > > left_join_non_hashtable_quals_
llvm::Value * codegenConstantWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
llvm::BasicBlock * nullcheck_bb
bool is_trivial_loop_join(const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
llvm::Value * element_count
llvm::Value * values_buffer
#define IS_EQUIVALENCE(X)
llvm::Value * codegenArith(const Analyzer::BinOper *, const CompilationOptions &)
std::set< int > aggregateResult(const std::set< int > &aggregate, const std::set< int > &next_result) const final
GroupColLLVMValue groupByColumnCodegen(Analyzer::Expr *group_by_col, const size_t col_width, const CompilationOptions &, const bool translate_null_val, const int64_t translated_null_val, DiamondCodegen &, std::stack< llvm::BasicBlock * > &, const bool thread_mem_shared)
bool with_dynamic_watchdog
llvm::IRBuilder ir_builder_
std::function< llvm::BasicBlock *(llvm::BasicBlock *, llvm::BasicBlock *, const std::string &, llvm::Function *, CgenState *)> HoistedFiltersCallback
llvm::Value * posArg(const Analyzer::Expr *) const
std::vector< InputDescriptor > input_descs
bool need_patch_unnest_double(const SQLTypeInfo &ti, const bool is_maxwell, const bool mem_shared)
std::set< int > visitColumnVar(const Analyzer::ColumnVar *col_expr) const final
llvm::ConstantInt * llBool(const bool v) const
virtual std::vector< llvm::Value * > codegenColumn(const Analyzer::ColumnVar *, const bool fetch_column, const CompilationOptions &)
void set_constant_expr() const
llvm::Value * codegenArrayAt(const Analyzer::BinOper *, const CompilationOptions &)
HOST DEVICE SQLTypes get_type() const
void setFalseTarget(llvm::BasicBlock *cond_false)
QualsConjunctiveForm qual_to_conjunctive_form(const std::shared_ptr< Analyzer::Expr > qual_expr)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::vector< llvm::Value * > codegenGeoBinOper(const Analyzer::GeoBinOper *, const CompilationOptions &)
std::shared_ptr< HashJoin > hash_table
double get_bound_val(const Analyzer::Expr *bound_expr) const
bool filter_on_deleted_column
llvm::Function * row_func_
llvm::Value * codegenIsNull(const Analyzer::UOper *, const CompilationOptions &)
SQLOps get_optype() const
std::set< int > visitUOper(const Analyzer::UOper *u_oper) const final
std::vector< llvm::Value * > codegenGeoExpr(const Analyzer::GeoExpr *, const CompilationOptions &)
llvm::LLVMContext & context_
llvm::Function * current_func_
llvm::Value * get_arg_by_name(llvm::Function *func, const std::string &name)
#define INJECT_TIMER(DESC)
const JoinQualsPerNestingLevel join_quals
std::vector< llvm::Value * > codegenGeoUOper(const Analyzer::GeoUOper *, const CompilationOptions &)
llvm::ConstantInt * inlineIntNull(const SQLTypeInfo &)
TableIdToNodeMap table_id_to_node_map
llvm::Value * codegenWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
llvm::Value * codegenFunctionOper(const Analyzer::FunctionOper *, const CompilationOptions &)
std::set< int > visitFunctionOper(const Analyzer::FunctionOper *func_expr) const final
static llvm::BasicBlock * codegen(const std::vector< JoinLoop > &join_loops, const std::function< llvm::BasicBlock *(const std::vector< llvm::Value * > &)> &body_codegen, llvm::Value *outer_iter, llvm::BasicBlock *exit_bb, CgenState *cgen_state)
void add_qualifier_to_execution_unit(RelAlgExecutionUnit &ra_exe_unit, const std::shared_ptr< Analyzer::Expr > &qual)
const std::vector< InputTableInfo > & query_infos_
ExpressionRange getExpressionRange(const Analyzer::BinOper *expr, const std::vector< InputTableInfo > &query_infos, const Executor *, boost::optional< std::list< std::shared_ptr< Analyzer::Expr >>> simple_quals)
std::vector< llvm::Value * > codegenArrayExpr(const Analyzer::ArrayExpr *, const CompilationOptions &)
llvm::BasicBlock * orig_cond_false_
const SQLTypeInfo & get_type_info() const
llvm::Value * codegenUMinus(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
llvm::Value * slot_lookup_result
std::set< int > visitBinOper(const Analyzer::BinOper *bin_oper) const final
ExecutorDeviceType device_type
std::unordered_map< int, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
const std::vector< InputTableInfo > & query_infos_
Expression class for string functions The "arg" constructor parameter must be an expression that reso...
std::shared_ptr< HashJoin > buildCurrentLevelHashTable(const JoinCondition ¤t_level_join_conditions, size_t level_idx, RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache, std::vector< std::string > &fail_reasons)
const Expr * get_arg() const
int32_t get_partition_count_val() const
static const int32_t ERR_WIDTH_BUCKET_INVALID_ARGUMENT
llvm::Value * toBool(llvm::Value *)
std::vector< llvm::Value * > codegenGeoColumnVar(const Analyzer::GeoColumnVar *, const bool fetch_columns, const CompilationOptions &co)
llvm::Value * codegenFunctionOperWithCustomTypeHandling(const Analyzer::FunctionOperWithCustomTypeHandling *, const CompilationOptions &)
llvm::Value * codegenCmp(const Analyzer::BinOper *, const CompilationOptions &)
std::list< std::shared_ptr< Analyzer::Expr > > getSimpleQuals() const
const Expr * get_target_value() const
std::list< std::shared_ptr< Analyzer::Expr > > quals
llvm::ConstantInt * llInt(const T v) const
llvm::Value * codegenUnnest(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * addJoinLoopIterator(const std::vector< llvm::Value * > &prev_iters, const size_t level_idx)
std::list< std::shared_ptr< Analyzer::Expr > > quals
RegisteredQueryHint query_hint
llvm::Value * finalize(llvm::Value *null_lv, llvm::Value *notnull_lv)
bool can_skip_out_of_bound_check() const
llvm::Value * codegenLogical(const Analyzer::BinOper *, const CompilationOptions &)
void check_if_loop_join_is_allowed(RelAlgExecutionUnit &ra_exe_unit, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, const size_t level_idx, const std::string &fail_reason)
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
llvm::ConstantInt * ll_bool(const bool v, llvm::LLVMContext &context)
JoinLoop::HoistedFiltersCallback buildHoistLeftHandSideFiltersCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const int inner_table_id, const CompilationOptions &co)
llvm::Value * codegenCast(const Analyzer::UOper *, const CompilationOptions &)
uint32_t log2_bytes(const uint32_t bytes)
std::string numeric_type_name(const SQLTypeInfo &ti)
bool is_dict_encoded_string() const
void skip_out_of_bound_check() const
void redeclareFilterFunction()
const Expr * get_lower_bound() const
std::vector< JoinLoop > buildJoinLoops(RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache)
std::function< llvm::Value *(const std::vector< llvm::Value * > &, llvm::Value *)> buildIsDeletedCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const CompilationOptions &co)
const Expr * get_upper_bound() const
bool allow_runtime_query_interrupt
llvm::ArrayType * get_int_array_type(int const width, int count, llvm::LLVMContext &context)
SQLOps get_optype() const
std::unique_ptr< DiamondCodegen > null_check
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals
HashTableBuildDagMap hash_table_build_plan_dag
llvm::ConstantFP * inlineFpNull(const SQLTypeInfo &)
void check_valid_join_qual(std::shared_ptr< Analyzer::BinOper > &bin_oper)
Executor * executor() const
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr(const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count)