17 #include "../Parser/ParserNode.h"
31 const bool fetch_columns,
57 if (ti.get_type() ==
kNULLT) {
58 throw std::runtime_error(
59 "NULL type literals are not currently supported in this context.");
61 if (constant->get_is_null()) {
63 ?
static_cast<llvm::Value*
>(
executor_->cgen_state_->inlineFpNull(ti))
64 : static_cast<llvm::Value*>(
executor_->cgen_state_->inlineIntNull(ti))};
69 CHECK_NE(ti.getStringDictKey().dict_id, 0);
70 return {
codegen(constant, ti.get_compression(), ti.getStringDictKey(), co)};
72 return {
codegen(constant, ti.get_compression(), {}, co)};
76 return {
codegen(case_expr, co)};
80 return {
codegen(extract_expr, co)};
84 return {
codegen(dateadd_expr, co)};
88 return {
codegen(datediff_expr, co)};
92 return {
codegen(datetrunc_expr, co)};
95 if (charlength_expr) {
96 return {
codegen(charlength_expr, co)};
99 if (keyforstring_expr) {
100 return {
codegen(keyforstring_expr, co)};
103 if (sample_ratio_expr) {
104 return {
codegen(sample_ratio_expr, co)};
107 if (string_oper_expr) {
108 return {
codegen(string_oper_expr, co)};
111 if (cardinality_expr) {
112 return {
codegen(cardinality_expr, co)};
116 return {
codegen(like_expr, co)};
120 return {
codegen(regexp_expr, co)};
127 if (likelihood_expr) {
128 return {
codegen(likelihood_expr->get_arg(), fetch_columns, co)};
135 if (in_integer_set_expr) {
136 return {
codegen(in_integer_set_expr, co)};
138 auto function_oper_with_custom_type_handling_expr =
140 if (function_oper_with_custom_type_handling_expr) {
142 function_oper_with_custom_type_handling_expr, co)};
145 if (array_oper_expr) {
157 if (function_oper_expr) {
164 if (dynamic_cast<const Analyzer::OffsetInFragment*>(expr)) {
167 if (dynamic_cast<const Analyzer::WindowFunction*>(expr)) {
220 auto input_expr = expr->
get_arg();
223 auto double_lv =
codegen(input_expr,
true, co);
224 CHECK_EQ(
size_t(1), double_lv.size());
226 std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
227 const bool is_nullable = !input_expr->get_type_info().get_notnull();
229 nullcheck_codegen = std::make_unique<NullCheckCodegen>(
cgen_state_,
232 input_expr->get_type_info(),
233 "sample_ratio_nullcheck");
236 std::vector<llvm::Value*>
args{double_lv[0],
posArg(
nullptr)};
238 if (nullcheck_codegen) {
251 CHECK(target_value_expr);
252 CHECK(lower_bound_expr);
253 CHECK(upper_bound_expr);
254 CHECK(partition_count_expr);
257 auto target_expr = expr;
258 if (
auto cast_expr = dynamic_cast<const Analyzer::UOper*>(expr)) {
260 target_expr = cast_expr->get_operand();
272 if (is_constant_expr(lower_bound_expr) && is_constant_expr(upper_bound_expr) &&
273 is_constant_expr(partition_count_expr)) {
287 if (!col_range.hasNulls()) {
312 if (num_partitions < 1 || num_partitions > INT32_MAX) {
313 throw std::runtime_error(
314 "PARTITION_COUNT expression of width_bucket function should be in a valid "
315 "range: 0 < PARTITION_COUNT <= 2147483647");
319 if (lower == upper) {
320 throw std::runtime_error(
321 "LOWER_BOUND and UPPER_BOUND expressions of width_bucket function cannot have "
322 "the same constant value");
325 throw std::runtime_error(
326 "Both LOWER_BOUND and UPPER_BOUND of width_bucket function should be finite "
327 "numeric constants.");
330 bool const reversed = lower > upper;
331 double const scale_factor = num_partitions / (reversed ? lower - upper : upper - lower);
332 std::string func_name = reversed ?
"width_bucket_reversed" :
"width_bucket";
334 auto get_double_constant_lvs = [
this, &co](
double const_val) {
337 auto double_const_expr =
339 return codegen(double_const_expr.get(),
false, co);
342 auto target_value_ti = target_value_expr->get_type_info();
343 auto target_value_expr_lvs =
codegen(target_value_expr,
true, co);
344 CHECK_EQ(
size_t(1), target_value_expr_lvs.size());
345 auto lower_expr_lvs =
codegen(lower_bound_expr,
true, co);
346 CHECK_EQ(
size_t(1), lower_expr_lvs.size());
347 auto scale_factor_lvs = get_double_constant_lvs(scale_factor);
348 CHECK_EQ(
size_t(1), scale_factor_lvs.size());
350 std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
353 func_name +=
"_no_oob_check";
354 width_bucket_args.push_back(scale_factor_lvs[0]);
356 auto upper_expr_lvs =
codegen(upper_bound_expr,
true, co);
357 CHECK_EQ(
size_t(1), upper_expr_lvs.size());
358 auto partition_count_expr_lvs =
codegen(partition_count_expr,
true, co);
359 CHECK_EQ(
size_t(1), partition_count_expr_lvs.size());
360 width_bucket_args.push_back(upper_expr_lvs[0]);
361 width_bucket_args.push_back(scale_factor_lvs[0]);
362 width_bucket_args.push_back(partition_count_expr_lvs[0]);
363 if (!target_value_ti.get_notnull()) {
364 func_name +=
"_nullable";
365 auto translated_null_value = target_value_ti.is_fp()
368 auto null_value_lvs = get_double_constant_lvs(translated_null_value);
369 CHECK_EQ(
size_t(1), null_value_lvs.size());
370 width_bucket_args.push_back(null_value_lvs[0]);
383 std::string func_name =
"width_bucket_expr";
384 bool nullable_expr =
false;
386 func_name +=
"_no_oob_check";
387 }
else if (!target_value_expr->get_type_info().get_notnull()) {
388 func_name +=
"_nullable";
389 nullable_expr =
true;
392 auto target_value_expr_lvs =
codegen(target_value_expr,
true, co);
393 CHECK_EQ(
size_t(1), target_value_expr_lvs.size());
394 auto lower_bound_expr_lvs =
codegen(lower_bound_expr,
true, co);
395 CHECK_EQ(
size_t(1), lower_bound_expr_lvs.size());
396 auto upper_bound_expr_lvs =
codegen(upper_bound_expr,
true, co);
397 CHECK_EQ(
size_t(1), upper_bound_expr_lvs.size());
398 auto partition_count_expr_lvs =
codegen(partition_count_expr,
true, co);
399 CHECK_EQ(
size_t(1), partition_count_expr_lvs.size());
400 auto target_value_ti = target_value_expr->get_type_info();
405 auto partition_count_ti = partition_count_expr->get_type_info();
406 CHECK(partition_count_ti.is_integer());
408 auto partition_count_expr_lv =
412 partition_count_ti.get_size() < int32_ti.get_size());
414 llvm::Value* partition_count_min =
416 llvm::BasicBlock* width_bucket_partition_count_ok_bb =
418 "width_bucket_partition_count_ok_bb",
420 llvm::BasicBlock* width_bucket_argument_check_fail_bb =
422 "width_bucket_argument_check_fail_bb",
425 width_bucket_argument_check_fail_bb,
426 width_bucket_partition_count_ok_bb);
432 llvm::BasicBlock* width_bucket_bound_check_ok_bb =
434 "width_bucket_bound_check_ok_bb",
436 llvm::Value* bound_check{
nullptr};
437 if (lower_bound_expr->get_type_info().get_notnull() &&
438 upper_bound_expr->get_type_info().get_notnull()) {
440 lower_bound_expr_lvs[0], upper_bound_expr_lvs[0],
"bound_check");
442 std::vector<llvm::Value*> bound_check_args{
443 lower_bound_expr_lvs[0],
444 upper_bound_expr_lvs[0],
450 bound_check, width_bucket_argument_check_fail_bb, width_bucket_bound_check_ok_bb);
455 lower_bound_expr_lvs,
456 lower_bound_expr->get_type_info(),
459 auto lower_bound_expr_lv = lower_bound_expr_lvs[0];
460 auto upper_bound_expr_lv = upper_bound_expr_lvs[0];
461 std::vector<llvm::Value*> width_bucket_args{target_value_expr_lvs[0],
465 partition_count_expr_lv};
467 width_bucket_args.push_back(null_value_lv);
475 const std::shared_ptr<Analyzer::Expr>& qual) {
478 qual_cf.simple_quals.begin(),
479 qual_cf.simple_quals.end());
480 ra_exe_unit.
quals.insert(
481 ra_exe_unit.
quals.end(), qual_cf.quals.begin(), qual_cf.quals.end());
486 const std::vector<InputTableInfo>& query_infos,
487 const size_t level_idx,
488 const std::string& fail_reason) {
492 throw std::runtime_error(
"Hash join failed, reason(s): " + fail_reason +
493 " | Incorrect # tables for executing loop join");
496 const bool has_loop_size_hint =
498 const size_t loop_join_size_threshold =
502 if (has_loop_size_hint && loop_join_size_threshold < loop_join_size) {
503 throw std::runtime_error(
504 "Hash join failed, reason(s): " + fail_reason +
505 " | Cannot fall back to loop join for non-trivial inner table size");
509 if (level_idx + 1 != ra_exe_unit.
join_quals.size()) {
510 throw std::runtime_error(
511 "Hash join failed, reason(s): " + fail_reason +
512 " | Cannot fall back to loop join for intermediate join quals");
514 if (loop_join_size_threshold < loop_join_size) {
515 throw std::runtime_error(
516 "Hash join failed, reason(s): " + fail_reason +
517 " | Cannot fall back to loop join for non-trivial inner table size");
520 throw std::runtime_error(
"Hash join failed, reason(s): " + fail_reason +
521 " | Loop join is disabled by user");
530 if (lhs_cv && rhs_cv && !bin_oper->is_overlaps_oper()) {
532 auto rhs_type = rhs_cv->get_type_info().get_type();
535 throw std::runtime_error(
536 "Join operation between full array columns (i.e., R.arr = S.arr) instead of "
537 "indexed array columns (i.e., R.arr[1] = S.arr[2]) is not supported yet.");
548 const std::vector<InputTableInfo>& query_infos,
552 std::vector<JoinLoop> join_loops;
553 for (
size_t level_idx = 0, current_hash_table_idx = 0;
556 const auto& current_level_join_conditions = ra_exe_unit.
join_quals[level_idx];
557 std::vector<std::string> fail_reasons;
558 const auto current_level_hash_table =
559 buildCurrentLevelHashTable(current_level_join_conditions,
566 const auto found_outer_join_matches_cb =
567 [
this, level_idx](llvm::Value* found_outer_join_matches) {
571 found_outer_join_matches;
573 const auto is_deleted_cb = buildIsDeletedCb(ra_exe_unit, level_idx, co);
574 auto rem_left_join_quals_it =
576 bool has_remaining_left_join_quals =
578 !rem_left_join_quals_it->second.empty();
579 const auto outer_join_condition_remaining_quals_cb =
580 [
this, level_idx, &co](
const std::vector<llvm::Value*>& prev_iters) {
585 addJoinLoopIterator(prev_iters, level_idx + 1);
590 for (
auto expr : it->second) {
594 code_generator.
codegen(expr.get(),
true, co).front()));
597 return left_join_cond;
599 if (current_level_hash_table) {
600 const auto hoisted_filters_cb = buildHoistLeftHandSideFiltersCb(
601 ra_exe_unit, level_idx, current_level_hash_table->getInnerTableId(), co);
603 join_loops.emplace_back(
605 current_level_join_conditions.type,
607 [
this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
608 const std::vector<llvm::Value*>& prev_iters) {
609 addJoinLoopIterator(prev_iters, level_idx);
612 current_level_hash_table->codegenSlot(co, current_hash_table_idx);
617 has_remaining_left_join_quals
618 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
619 outer_join_condition_remaining_quals_cb)
622 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
627 }
else if (
auto range_join_table =
628 dynamic_cast<RangeJoinHashTable*>(current_level_hash_table.get())) {
629 join_loops.emplace_back(
631 current_level_join_conditions.type,
635 current_hash_table_idx,
637 current_level_hash_table,
638 &co](
const std::vector<llvm::Value*>& prev_iters) {
639 addJoinLoopIterator(prev_iters, level_idx);
641 CHECK(!prev_iters.empty());
642 const auto matching_set = range_join_table->codegenMatchingSetWithOffset(
643 co, current_hash_table_idx, prev_iters.back());
644 domain.values_buffer = matching_set.elements;
645 domain.element_count = matching_set.count;
650 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
651 outer_join_condition_remaining_quals_cb)
655 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
661 join_loops.emplace_back(
663 current_level_join_conditions.type,
665 [
this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
666 const std::vector<llvm::Value*>& prev_iters) {
667 addJoinLoopIterator(prev_iters, level_idx);
669 const auto matching_set = current_level_hash_table->codegenMatchingSet(
670 co, current_hash_table_idx);
672 domain.element_count = matching_set.count;
677 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
678 outer_join_condition_remaining_quals_cb)
681 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
687 ++current_hash_table_idx;
689 const auto fail_reasons_str = current_level_join_conditions.quals.empty()
690 ?
"No equijoin expression found"
693 ra_exe_unit, eo, query_infos, level_idx, fail_reasons_str);
696 VLOG(1) <<
"Unable to build hash table, falling back to loop join: "
698 const auto outer_join_condition_cb =
699 [
this, level_idx, &co, ¤t_level_join_conditions](
700 const std::vector<llvm::Value*>& prev_iters) {
705 addJoinLoopIterator(prev_iters, level_idx + 1);
708 for (
auto expr : current_level_join_conditions.quals) {
711 code_generator.toBool(
712 code_generator.codegen(expr.get(),
true, co).front()));
714 return left_join_cond;
716 join_loops.emplace_back(
718 current_level_join_conditions.type,
720 [
this, level_idx](
const std::vector<llvm::Value*>& prev_iters) {
721 addJoinLoopIterator(prev_iters, level_idx);
725 arg->getType()->getScalarType()->getPointerElementType(),
729 rows_per_scan_ptr->getType()->getPointerElementType(),
731 "num_rows_per_scan");
736 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
737 outer_join_condition_cb)
741 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
757 return {col_expr->getTableKey()};
762 std::set<shared::TableKey> ret;
763 for (
size_t i = 0; i < func_expr->getArity(); i++) {
764 ret = aggregateResult(ret, visit(func_expr->getArg(i)));
770 std::set<shared::TableKey> ret;
771 ret = aggregateResult(ret, visit(bin_oper->get_left_operand()));
772 return aggregateResult(ret, visit(bin_oper->get_right_operand()));
776 return visit(u_oper->get_operand());
780 const std::set<shared::TableKey>& aggregate,
781 const std::set<shared::TableKey>& next_result)
const final {
782 auto ret = aggregate;
783 for (
const auto& el : next_result) {
794 const size_t level_idx,
801 const auto& current_level_join_conditions = ra_exe_unit.
join_quals[level_idx];
802 if (level_idx == 0 && current_level_join_conditions.type ==
JoinType::LEFT) {
803 const auto& condition = current_level_join_conditions.quals.front();
805 CHECK(bin_oper) << condition->toString();
810 if (lhs && rhs && lhs->getTableKey() != rhs->getTableKey()) {
815 if (lhs->getTableKey() == inner_table_id) {
817 }
else if (rhs->getTableKey() == inner_table_id) {
821 std::list<std::shared_ptr<Analyzer::Expr>> hoisted_quals;
823 auto should_hoist_qual = [&hoisted_quals](
const auto& qual,
827 ExprTableIdVisitor visitor;
828 const auto table_keys = visitor.visit(qual.get());
829 if (table_keys.size() == 1 && table_keys.find(table_key) != table_keys.end()) {
830 hoisted_quals.push_back(qual);
834 should_hoist_qual(qual, selected_lhs->getTableKey());
836 for (
const auto& qual : ra_exe_unit.
quals) {
837 should_hoist_qual(qual, selected_lhs->getTableKey());
841 if (!hoisted_quals.empty()) {
842 return [
this, hoisted_quals, co](llvm::BasicBlock* true_bb,
843 llvm::BasicBlock* exit_bb,
844 const std::string& loop_name,
845 llvm::Function* parent_func,
846 CgenState* cgen_state) -> llvm::BasicBlock* {
848 bool has_quals_to_hoist =
false;
849 for (
const auto& qual : hoisted_quals) {
852 if (plan_state_->hoisted_filters_.count(qual) == 0) {
853 has_quals_to_hoist =
true;
858 if (!has_quals_to_hoist) {
864 llvm::IRBuilder<>& builder = cgen_state->ir_builder_;
865 auto& context = builder.getContext();
867 const auto filter_bb =
868 llvm::BasicBlock::Create(context,
869 "hoisted_left_join_filters_" + loop_name,
872 builder.SetInsertPoint(filter_bb);
874 llvm::Value* filter_lv = cgen_state_->llBool(
true);
877 for (
const auto& qual : hoisted_quals) {
878 if (plan_state_->hoisted_filters_.insert(qual).second) {
881 VLOG(1) <<
"Generating code for hoisted left hand side qualifier "
883 auto cond = code_generator.
toBool(
884 code_generator.
codegen(qual.get(),
true, co).front());
885 filter_lv = builder.CreateAnd(filter_lv, cond);
888 CHECK(filter_lv->getType()->isIntegerTy(1));
890 builder.CreateCondBr(filter_lv, true_bb, exit_bb);
900 std::function<llvm::Value*(const std::vector<llvm::Value*>&, llvm::Value*)>
902 const size_t level_idx,
909 const auto input_desc = ra_exe_unit.
input_descs[level_idx + 1];
914 const auto deleted_cd = plan_state_->getDeletedColForTable(input_desc.getTableKey());
918 CHECK(deleted_cd->columnType.is_boolean());
919 const auto deleted_expr = makeExpr<Analyzer::ColumnVar>(
920 deleted_cd->columnType,
922 input_desc.getNestLevel());
923 return [
this, deleted_expr, level_idx, &co](
const std::vector<llvm::Value*>& prev_iters,
924 llvm::Value* have_more_inner_rows) {
925 const auto matching_row_index = addJoinLoopIterator(prev_iters, level_idx + 1);
929 llvm::Value* is_valid_it{
nullptr};
930 if (have_more_inner_rows) {
931 is_valid_it = have_more_inner_rows;
933 is_valid_it = cgen_state_->ir_builder_.CreateICmp(
934 llvm::ICmpInst::ICMP_SGE, matching_row_index, cgen_state_->llInt<int64_t>(0));
936 const auto it_valid_bb = llvm::BasicBlock::Create(
937 cgen_state_->context_,
"it_valid", cgen_state_->current_func_);
938 const auto it_not_valid_bb = llvm::BasicBlock::Create(
939 cgen_state_->context_,
"it_not_valid", cgen_state_->current_func_);
940 cgen_state_->ir_builder_.CreateCondBr(is_valid_it, it_valid_bb, it_not_valid_bb);
941 const auto row_is_deleted_bb = llvm::BasicBlock::Create(
942 cgen_state_->context_,
"row_is_deleted", cgen_state_->current_func_);
943 cgen_state_->ir_builder_.SetInsertPoint(it_valid_bb);
945 const auto row_is_deleted = code_generator.
toBool(
946 code_generator.
codegen(deleted_expr.get(),
true, co).front());
947 cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
948 cgen_state_->ir_builder_.SetInsertPoint(it_not_valid_bb);
949 const auto row_is_deleted_default = cgen_state_->llBool(
false);
950 cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
951 cgen_state_->ir_builder_.SetInsertPoint(row_is_deleted_bb);
952 auto row_is_deleted_or_default =
953 cgen_state_->ir_builder_.CreatePHI(row_is_deleted->getType(), 2);
954 row_is_deleted_or_default->addIncoming(row_is_deleted, it_valid_bb);
955 row_is_deleted_or_default->addIncoming(row_is_deleted_default, it_not_valid_bb);
956 return row_is_deleted_or_default;
965 const std::vector<InputTableInfo>& query_infos,
967 std::vector<std::string>& fail_reasons) {
969 std::shared_ptr<HashJoin> current_level_hash_table;
970 auto handleNonHashtableQual = [&ra_exe_unit, &level_idx,
this](
972 std::shared_ptr<Analyzer::Expr> qual) {
974 plan_state_->addNonHashtableQualForLeftJoin(level_idx, qual);
979 for (
const auto& join_qual : current_level_join_conditions.
quals) {
981 if (current_level_hash_table || !qual_bin_oper ||
983 handleNonHashtableQual(current_level_join_conditions.
type, join_qual);
984 if (!current_level_hash_table) {
985 fail_reasons.emplace_back(
"No equijoin expression found");
991 if (!current_level_hash_table) {
992 hash_table_or_error = buildHashTableForQualifier(
997 current_level_join_conditions.
type,
1003 current_level_hash_table = hash_table_or_error.
hash_table;
1006 plan_state_->join_info_.join_hash_tables_.push_back(hash_table_or_error.
hash_table);
1007 plan_state_->join_info_.equi_join_tautologies_.push_back(qual_bin_oper);
1009 fail_reasons.push_back(hash_table_or_error.
fail_reason);
1010 if (!current_level_hash_table) {
1011 VLOG(2) <<
"Building a hashtable based on a qual " << qual_bin_oper->toString()
1014 handleNonHashtableQual(current_level_join_conditions.
type, qual_bin_oper);
1017 return current_level_hash_table;
1021 if (!cgen_state_->filter_func_) {
1029 for (
auto bb_it = cgen_state_->filter_func_->begin();
1030 bb_it != cgen_state_->filter_func_->end();
1032 for (
auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1034 for (
auto op_it = instr_it->value_op_begin(); op_it != instr_it->value_op_end();
1036 llvm::Value* v = *op_it;
1040 if (llvm::dyn_cast<const llvm::CallInst>(instr_it) &&
1041 op_it == instr_it->value_op_end() - 1) {
1046 if (
auto* instr = llvm::dyn_cast<llvm::Instruction>(v);
1047 instr && instr->getParent() &&
1048 instr->getParent()->getParent() == cgen_state_->row_func_) {
1050 cgen_state_->filter_func_args_[v] =
nullptr;
1051 }
else if (
auto* argum = llvm::dyn_cast<llvm::Argument>(v);
1052 argum && argum->getParent() == cgen_state_->row_func_) {
1054 cgen_state_->filter_func_args_[v] =
nullptr;
1062 std::vector<llvm::Type*> filter_func_arg_types;
1063 filter_func_arg_types.reserve(cgen_state_->filter_func_args_.v_.size());
1064 for (
auto& arg : cgen_state_->filter_func_args_.v_) {
1065 filter_func_arg_types.push_back(arg->getType());
1067 auto ft = llvm::FunctionType::get(
1068 get_int_type(32, cgen_state_->context_), filter_func_arg_types,
false);
1069 cgen_state_->filter_func_->setName(
"old_filter_func");
1070 auto filter_func2 = llvm::Function::Create(ft,
1071 llvm::Function::ExternalLinkage,
1073 cgen_state_->filter_func_->getParent());
1074 CHECK_EQ(filter_func2->arg_size(), cgen_state_->filter_func_args_.v_.size());
1075 auto arg_it = cgen_state_->filter_func_args_.begin();
1077 for (llvm::Function::arg_iterator I = filter_func2->arg_begin(),
1078 E = filter_func2->arg_end();
1081 arg_it->second = &*I;
1082 if (arg_it->first->hasName()) {
1083 I->setName(arg_it->first->getName());
1092 filter_func2->getBasicBlockList().splice(
1093 filter_func2->begin(), cgen_state_->filter_func_->getBasicBlockList());
1095 if (cgen_state_->current_func_ == cgen_state_->filter_func_) {
1096 cgen_state_->current_func_ = filter_func2;
1098 cgen_state_->filter_func_ = filter_func2;
1101 for (
auto bb_it = cgen_state_->filter_func_->begin();
1102 bb_it != cgen_state_->filter_func_->end();
1104 for (
auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
1106 for (
auto op_it = instr_it->op_begin(); op_it != instr_it->op_end(); ++op_it, ++i) {
1107 llvm::Value* v = op_it->get();
1108 if (
auto arg_it = cgen_state_->filter_func_args_.find(v);
1109 arg_it != cgen_state_->filter_func_args_.end()) {
1111 llvm::Use* use = &*op_it;
1112 use->set(arg_it->second);
1120 const size_t level_idx) {
1125 const auto it = cgen_state_->scan_idx_to_hash_pos_.find(level_idx);
1126 if (it != cgen_state_->scan_idx_to_hash_pos_.end()) {
1129 CHECK(!prev_iters.empty());
1130 llvm::Value* matching_row_index = prev_iters.back();
1132 cgen_state_->scan_idx_to_hash_pos_.emplace(level_idx, matching_row_index);
1133 CHECK(it_ok.second);
1134 return matching_row_index;
1140 llvm::Function* query_func,
1141 llvm::BasicBlock* entry_bb,
1146 const auto exit_bb =
1147 llvm::BasicBlock::Create(cgen_state_->context_,
"exit", cgen_state_->current_func_);
1148 cgen_state_->ir_builder_.SetInsertPoint(exit_bb);
1149 cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
1150 cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1153 llvm::BasicBlock* loops_entry_bb{
nullptr};
1154 auto has_range_join =
1155 std::any_of(join_loops.begin(), join_loops.end(), [](
const auto& join_loop) {
1158 if (has_range_join) {
1159 CHECK_EQ(join_loops.size(), size_t(1));
1160 const auto element_count =
1161 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_), 9);
1163 auto compute_packed_offset = [](
const int32_t x,
const int32_t y) -> uint64_t {
1164 const uint64_t y_shifted =
static_cast<uint64_t
>(y) << 32;
1165 return y_shifted |
static_cast<uint32_t
>(x);
1168 const auto values_arr = std::vector<llvm::Constant*>{
1169 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_), 0),
1170 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1171 compute_packed_offset(0, 1)),
1172 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1173 compute_packed_offset(0, -1)),
1174 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1175 compute_packed_offset(1, 0)),
1176 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1177 compute_packed_offset(1, 1)),
1178 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1179 compute_packed_offset(1, -1)),
1180 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1181 compute_packed_offset(-1, 0)),
1182 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1183 compute_packed_offset(-1, 1)),
1184 llvm::ConstantInt::get(
get_int_type(64, cgen_state_->context_),
1185 compute_packed_offset(-1, -1))};
1187 const auto constant_values_array = llvm::ConstantArray::get(
1189 CHECK(cgen_state_->module_);
1191 new llvm::GlobalVariable(*cgen_state_->module_,
1194 llvm::GlobalValue::LinkageTypes::InternalLinkage,
1195 constant_values_array);
1199 [element_count, values](
const std::vector<llvm::Value*>& v) {
1202 domain.values_buffer = values;
1218 &group_by_and_aggregate,
1220 &ra_exe_unit](
const std::vector<llvm::Value*>& prev_iters) {
1221 auto& builder = cgen_state_->ir_builder_;
1224 llvm::BasicBlock::Create(cgen_state_->context_,
1225 "range_key_inner_body_exit",
1226 builder.GetInsertBlock()->getParent());
1228 auto range_key_body_bb =
1229 llvm::BasicBlock::Create(cgen_state_->context_,
1230 "range_key_loop_body",
1231 builder.GetInsertBlock()->getParent());
1232 builder.SetInsertPoint(range_key_body_bb);
1241 &group_by_and_aggregate,
1243 &ra_exe_unit](
const std::vector<llvm::Value*>& prev_iters) {
1244 addJoinLoopIterator(prev_iters, join_loops.size());
1245 auto& builder = cgen_state_->ir_builder_;
1246 const auto loop_body_bb =
1247 llvm::BasicBlock::Create(builder.getContext(),
1249 builder.GetInsertBlock()->getParent());
1250 builder.SetInsertPoint(loop_body_bb);
1251 const bool can_return_error =
1252 compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1253 if (can_return_error || cgen_state_->needs_error_check_ ||
1254 eo.with_dynamic_watchdog || eo.allow_runtime_query_interrupt) {
1255 createErrorCheckControlFlow(query_func,
1256 eo.with_dynamic_watchdog,
1257 eo.allow_runtime_query_interrupt,
1260 group_by_and_aggregate.query_infos_);
1262 return loop_body_bb;
1268 builder.SetInsertPoint(range_key_body_bb);
1269 cgen_state_->ir_builder_.CreateBr(body_loops_entry_bb);
1271 builder.SetInsertPoint(body_exit_bb);
1272 return range_key_body_bb;
1274 code_generator.
posArg(
nullptr),
1286 &group_by_and_aggregate,
1288 &ra_exe_unit](
const std::vector<llvm::Value*>& prev_iters) {
1290 addJoinLoopIterator(prev_iters, join_loops.size());
1291 auto& builder = cgen_state_->ir_builder_;
1292 const auto loop_body_bb = llvm::BasicBlock::Create(
1293 builder.getContext(),
"loop_body", builder.GetInsertBlock()->getParent());
1294 builder.SetInsertPoint(loop_body_bb);
1295 const bool can_return_error =
1296 compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
1297 if (can_return_error || cgen_state_->needs_error_check_ ||
1299 createErrorCheckControlFlow(query_func,
1306 return loop_body_bb;
1308 code_generator.
posArg(
nullptr),
1312 CHECK(loops_entry_bb);
1313 cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
1314 cgen_state_->ir_builder_.CreateBr(loops_entry_bb);
1319 const size_t col_width,
1321 const bool translate_null_val,
1322 const int64_t translated_null_val,
1324 std::stack<llvm::BasicBlock*>& array_loops,
1325 const bool thread_mem_shared) {
1327 CHECK_GE(col_width,
sizeof(int32_t));
1329 auto group_key = code_generator.
codegen(group_by_col,
true, co).front();
1330 auto key_to_cache = group_key;
1331 if (dynamic_cast<Analyzer::UOper*>(group_by_col) &&
1332 static_cast<Analyzer::UOper*>(group_by_col)->get_optype() ==
kUNNEST) {
1333 auto preheader = cgen_state_->ir_builder_.GetInsertBlock();
1334 auto array_loop_head = llvm::BasicBlock::Create(cgen_state_->context_,
1336 cgen_state_->current_func_,
1337 preheader->getNextNode());
1339 const auto ret_ty =
get_int_type(32, cgen_state_->context_);
1340 auto array_idx_ptr = cgen_state_->ir_builder_.CreateAlloca(ret_ty);
1341 CHECK(array_idx_ptr);
1342 cgen_state_->ir_builder_.CreateStore(cgen_state_->llInt(int32_t(0)), array_idx_ptr);
1343 const auto arr_expr =
static_cast<Analyzer::UOper*
>(group_by_col)->get_operand();
1345 CHECK(array_ti.is_array());
1346 const auto& elem_ti = array_ti.get_elem_type();
1348 (array_ti.get_size() > 0)
1349 ? cgen_state_->llInt(array_ti.get_size() / elem_ti.get_size())
1350 : cgen_state_->emitExternalCall(
1354 code_generator.
posArg(arr_expr),
1355 cgen_state_->llInt(
log2_bytes(elem_ti.get_logical_size()))});
1356 cgen_state_->ir_builder_.CreateBr(array_loop_head);
1357 cgen_state_->ir_builder_.SetInsertPoint(array_loop_head);
1359 auto array_idx = cgen_state_->ir_builder_.CreateLoad(
1360 array_idx_ptr->getType()->getPointerElementType(), array_idx_ptr);
1361 auto bound_check = cgen_state_->ir_builder_.CreateICmp(
1362 llvm::ICmpInst::ICMP_SLT, array_idx, array_len);
1363 auto array_loop_body = llvm::BasicBlock::Create(
1364 cgen_state_->context_,
"array_loop_body", cgen_state_->current_func_);
1365 cgen_state_->ir_builder_.CreateCondBr(
1368 array_loops.empty() ? diamond_codegen.
orig_cond_false_ : array_loops.top());
1369 cgen_state_->ir_builder_.SetInsertPoint(array_loop_body);
1370 cgen_state_->ir_builder_.CreateStore(
1371 cgen_state_->ir_builder_.CreateAdd(array_idx, cgen_state_->llInt(int32_t(1))),
1374 if (array_ti.get_size() < 0) {
1375 if (array_ti.get_notnull()) {
1376 array_at_fname =
"notnull_" + array_at_fname;
1378 array_at_fname =
"varlen_" + array_at_fname;
1380 const auto ar_ret_ty =
1382 ? (elem_ti.get_type() ==
kDOUBLE
1383 ? llvm::Type::getDoubleTy(cgen_state_->context_)
1384 : llvm::Type::getFloatTy(cgen_state_->context_))
1385 :
get_int_type(elem_ti.get_logical_size() * 8, cgen_state_->context_);
1386 group_key = cgen_state_->emitExternalCall(
1389 {group_key, code_generator.
posArg(arr_expr), array_idx});
1391 elem_ti, isArchMaxwell(co.
device_type), thread_mem_shared)) {
1392 key_to_cache = spillDoubleElement(group_key, ar_ret_ty);
1394 key_to_cache = group_key;
1396 CHECK(array_loop_head);
1397 array_loops.push(array_loop_head);
1399 cgen_state_->group_by_expr_cache_.push_back(key_to_cache);
1400 llvm::Value* orig_group_key{
nullptr};
1401 if (translate_null_val) {
1402 const std::string translator_func_name(
1403 col_width ==
sizeof(int32_t) ?
"translate_null_key_i32_" :
"translate_null_key_");
1404 const auto& ti = group_by_col->get_type_info();
1405 const auto key_type =
get_int_type(ti.get_logical_size() * 8, cgen_state_->context_);
1406 orig_group_key = group_key;
1407 group_key = cgen_state_->emitCall(
1410 static_cast<llvm::Value*
>(
1412 static_cast<llvm::Value*>(llvm::ConstantInt::get(
1413 llvm::Type::getInt64Ty(cgen_state_->context_), translated_null_val))});
1415 group_key = cgen_state_->ir_builder_.CreateBitCast(
1416 cgen_state_->castToTypeIn(group_key, col_width * 8),
1418 if (orig_group_key) {
1419 orig_group_key = cgen_state_->ir_builder_.CreateBitCast(
1420 cgen_state_->castToTypeIn(orig_group_key, col_width * 8),
1423 return {group_key, orig_group_key};
1428 llvm::Value* nullable_lv,
1430 const std::string&
name)
1431 : cgen_state(cgen_state), name(name) {
1436 llvm::Value* is_null_lv{
nullptr};
1437 if (nullable_ti.
is_fp()) {
1439 llvm::FCmpInst::FCMP_OEQ, nullable_lv, cgen_state->
inlineFpNull(nullable_ti));
1441 nullable_lv->getType()->getIntegerBitWidth() == 1) {
1443 llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->
llBool(
true));
1446 llvm::ICmpInst::ICMP_EQ, nullable_lv, cgen_state->
inlineIntNull(nullable_ti));
1450 std::make_unique<DiamondCodegen>(is_null_lv,
executor,
false,
name,
nullptr,
false);
1464 llvm::Value* notnull_lv) {
1467 cgen_state->ir_builder_.CreateBr(nullcheck_bb);
1469 CHECK_EQ(null_lv->getType(), notnull_lv->getType());
1471 cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1473 cgen_state->ir_builder_.CreatePHI(null_lv->getType(), 2,
name +
"_value");
1474 nullcheck_value->addIncoming(notnull_lv, null_check->cond_false_);
1475 nullcheck_value->addIncoming(null_lv, null_check->cond_true_);
1477 null_check.reset(
nullptr);
1478 cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
1479 return nullcheck_value;
bool g_enable_left_join_filter_hoisting
NullCheckCodegen(CgenState *cgen_state, Executor *executor, llvm::Value *nullable_lv, const SQLTypeInfo &nullable_ti, const std::string &name="")
void codegenJoinLoops(const std::vector< JoinLoop > &join_loops, const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, llvm::Function *query_func, llvm::BasicBlock *entry_bb, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const ExecutionOptions &eo)
const Expr * get_partition_count() const
std::vector< llvm::Value * > outer_join_match_found_per_level_
bool is_constant_expr() const
std::unordered_map< size_t, std::vector< std::shared_ptr< Analyzer::Expr > > > left_join_non_hashtable_quals_
llvm::Value * codegenConstantWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
llvm::BasicBlock * nullcheck_bb
llvm::Value * element_count
llvm::Value * values_buffer
#define IS_EQUIVALENCE(X)
llvm::Value * codegenArith(const Analyzer::BinOper *, const CompilationOptions &)
GroupColLLVMValue groupByColumnCodegen(Analyzer::Expr *group_by_col, const size_t col_width, const CompilationOptions &, const bool translate_null_val, const int64_t translated_null_val, DiamondCodegen &, std::stack< llvm::BasicBlock * > &, const bool thread_mem_shared)
bool with_dynamic_watchdog
llvm::IRBuilder ir_builder_
std::function< llvm::BasicBlock *(llvm::BasicBlock *, llvm::BasicBlock *, const std::string &, llvm::Function *, CgenState *)> HoistedFiltersCallback
llvm::Value * posArg(const Analyzer::Expr *) const
std::vector< InputDescriptor > input_descs
bool need_patch_unnest_double(const SQLTypeInfo &ti, const bool is_maxwell, const bool mem_shared)
llvm::ConstantInt * llBool(const bool v) const
virtual std::vector< llvm::Value * > codegenColumn(const Analyzer::ColumnVar *, const bool fetch_column, const CompilationOptions &)
void set_constant_expr() const
unsigned g_trivial_loop_join_threshold
llvm::Value * codegenArrayAt(const Analyzer::BinOper *, const CompilationOptions &)
HOST DEVICE SQLTypes get_type() const
void setFalseTarget(llvm::BasicBlock *cond_false)
QualsConjunctiveForm qual_to_conjunctive_form(const std::shared_ptr< Analyzer::Expr > qual_expr)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::vector< llvm::Value * > codegenGeoBinOper(const Analyzer::GeoBinOper *, const CompilationOptions &)
std::shared_ptr< HashJoin > hash_table
double get_bound_val(const Analyzer::Expr *bound_expr) const
bool filter_on_deleted_column
llvm::Function * row_func_
llvm::Value * codegenIsNull(const Analyzer::UOper *, const CompilationOptions &)
SQLOps get_optype() const
std::vector< llvm::Value * > codegenGeoExpr(const Analyzer::GeoExpr *, const CompilationOptions &)
llvm::LLVMContext & context_
llvm::Function * current_func_
llvm::Value * get_arg_by_name(llvm::Function *func, const std::string &name)
std::set< shared::TableKey > visitFunctionOper(const Analyzer::FunctionOper *func_expr) const final
#define INJECT_TIMER(DESC)
const JoinQualsPerNestingLevel join_quals
std::vector< llvm::Value * > codegenGeoUOper(const Analyzer::GeoUOper *, const CompilationOptions &)
llvm::ConstantInt * inlineIntNull(const SQLTypeInfo &)
TableIdToNodeMap table_id_to_node_map
llvm::Value * codegenWidthBucketExpr(const Analyzer::WidthBucketExpr *, const CompilationOptions &)
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
llvm::Value * codegenFunctionOper(const Analyzer::FunctionOper *, const CompilationOptions &)
static llvm::BasicBlock * codegen(const std::vector< JoinLoop > &join_loops, const std::function< llvm::BasicBlock *(const std::vector< llvm::Value * > &)> &body_codegen, llvm::Value *outer_iter, llvm::BasicBlock *exit_bb, CgenState *cgen_state)
void add_qualifier_to_execution_unit(RelAlgExecutionUnit &ra_exe_unit, const std::shared_ptr< Analyzer::Expr > &qual)
const std::vector< InputTableInfo > & query_infos_
ExpressionRange getExpressionRange(const Analyzer::BinOper *expr, const std::vector< InputTableInfo > &query_infos, const Executor *, boost::optional< std::list< std::shared_ptr< Analyzer::Expr >>> simple_quals)
std::vector< llvm::Value * > codegenArrayExpr(const Analyzer::ArrayExpr *, const CompilationOptions &)
llvm::BasicBlock * orig_cond_false_
const SQLTypeInfo & get_type_info() const
llvm::Value * codegenUMinus(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
llvm::Value * slot_lookup_result
ExecutorDeviceType device_type
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
const std::vector< InputTableInfo > & query_infos_
Expression class for string functions The "arg" constructor parameter must be an expression that reso...
std::set< shared::TableKey > aggregateResult(const std::set< shared::TableKey > &aggregate, const std::set< shared::TableKey > &next_result) const final
std::shared_ptr< HashJoin > buildCurrentLevelHashTable(const JoinCondition ¤t_level_join_conditions, size_t level_idx, RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache, std::vector< std::string > &fail_reasons)
bool isHintRegistered(const QueryHint hint) const
const Expr * get_arg() const
std::set< shared::TableKey > visitBinOper(const Analyzer::BinOper *bin_oper) const final
std::unordered_map< shared::TableKey, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
std::set< shared::TableKey > visitUOper(const Analyzer::UOper *u_oper) const final
int32_t get_partition_count_val() const
static const int32_t ERR_WIDTH_BUCKET_INVALID_ARGUMENT
llvm::Value * toBool(llvm::Value *)
std::vector< llvm::Value * > codegenGeoColumnVar(const Analyzer::GeoColumnVar *, const bool fetch_columns, const CompilationOptions &co)
llvm::Value * codegenFunctionOperWithCustomTypeHandling(const Analyzer::FunctionOperWithCustomTypeHandling *, const CompilationOptions &)
llvm::Value * codegenCmp(const Analyzer::BinOper *, const CompilationOptions &)
std::list< std::shared_ptr< Analyzer::Expr > > getSimpleQuals() const
const Expr * get_target_value() const
std::list< std::shared_ptr< Analyzer::Expr > > quals
llvm::ConstantInt * llInt(const T v) const
llvm::Value * codegenUnnest(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * addJoinLoopIterator(const std::vector< llvm::Value * > &prev_iters, const size_t level_idx)
std::list< std::shared_ptr< Analyzer::Expr > > quals
RegisteredQueryHint query_hint
llvm::Value * finalize(llvm::Value *null_lv, llvm::Value *notnull_lv)
bool can_skip_out_of_bound_check() const
llvm::Value * codegenLogical(const Analyzer::BinOper *, const CompilationOptions &)
void check_if_loop_join_is_allowed(RelAlgExecutionUnit &ra_exe_unit, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, const size_t level_idx, const std::string &fail_reason)
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
llvm::ConstantInt * ll_bool(const bool v, llvm::LLVMContext &context)
size_t loop_join_inner_table_max_num_rows
llvm::Value * codegenCast(const Analyzer::UOper *, const CompilationOptions &)
uint32_t log2_bytes(const uint32_t bytes)
std::string numeric_type_name(const SQLTypeInfo &ti)
bool is_dict_encoded_string() const
void skip_out_of_bound_check() const
void redeclareFilterFunction()
bool any_of(std::vector< Analyzer::Expr * > const &target_exprs)
const Expr * get_lower_bound() const
std::vector< JoinLoop > buildJoinLoops(RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache)
std::function< llvm::Value *(const std::vector< llvm::Value * > &, llvm::Value *)> buildIsDeletedCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const CompilationOptions &co)
JoinLoop::HoistedFiltersCallback buildHoistLeftHandSideFiltersCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const shared::TableKey &inner_table_key, const CompilationOptions &co)
const Expr * get_upper_bound() const
bool allow_runtime_query_interrupt
llvm::ArrayType * get_int_array_type(int const width, int count, llvm::LLVMContext &context)
SQLOps get_optype() const
std::unique_ptr< DiamondCodegen > null_check
std::set< shared::TableKey > visitColumnVar(const Analyzer::ColumnVar *col_expr) const final
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals
HashTableBuildDagMap hash_table_build_plan_dag
llvm::ConstantFP * inlineFpNull(const SQLTypeInfo &)
void check_valid_join_qual(std::shared_ptr< Analyzer::BinOper > &bin_oper)
Executor * executor() const
size_t get_loop_join_size(const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr(const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count)