17 #include "../Parser/ParserNode.h"
27 const bool fetch_columns,
48 if (ti.get_type() ==
kNULLT) {
49 throw std::runtime_error(
50 "NULL type literals are not currently supported in this context.");
52 if (constant->get_is_null()) {
54 ?
static_cast<llvm::Value*
>(
executor_->cgen_state_->inlineFpNull(ti))
55 : static_cast<llvm::Value*>(
executor_->cgen_state_->inlineIntNull(ti))};
61 return {
codegen(constant, ti.get_compression(), ti.get_comp_param(), co)};
63 return {
codegen(constant, ti.get_compression(), 0, co)};
67 return {
codegen(case_expr, co)};
71 return {
codegen(extract_expr, co)};
75 return {
codegen(dateadd_expr, co)};
79 return {
codegen(datediff_expr, co)};
83 return {
codegen(datetrunc_expr, co)};
86 if (charlength_expr) {
87 return {
codegen(charlength_expr, co)};
90 if (keyforstring_expr) {
91 return {
codegen(keyforstring_expr, co)};
94 if (sample_ratio_expr) {
95 return {
codegen(sample_ratio_expr, co)};
99 return {
codegen(lower_expr, co)};
102 if (cardinality_expr) {
103 return {
codegen(cardinality_expr, co)};
107 return {
codegen(like_expr, co)};
111 return {
codegen(regexp_expr, co)};
114 if (likelihood_expr) {
115 return {
codegen(likelihood_expr->get_arg(), fetch_columns, co)};
122 if (in_integer_set_expr) {
123 return {
codegen(in_integer_set_expr, co)};
125 auto function_oper_with_custom_type_handling_expr =
127 if (function_oper_with_custom_type_handling_expr) {
129 function_oper_with_custom_type_handling_expr, co)};
132 if (array_oper_expr) {
144 if (function_oper_expr) {
147 if (dynamic_cast<const Analyzer::OffsetInFragment*>(expr)) {
150 if (dynamic_cast<const Analyzer::WindowFunction*>(expr)) {
202 auto input_expr = expr->
get_arg();
205 auto double_lv =
codegen(input_expr,
true, co);
206 CHECK_EQ(
size_t(1), double_lv.size());
208 std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
209 const bool is_nullable = !input_expr->get_type_info().get_notnull();
211 nullcheck_codegen = std::make_unique<NullCheckCodegen>(
cgen_state_,
214 input_expr->get_type_info(),
215 "sample_ratio_nullcheck");
218 std::vector<llvm::Value*>
args{double_lv[0],
posArg(
nullptr)};
220 if (nullcheck_codegen) {
229 const std::shared_ptr<Analyzer::Expr>& qual) {
232 qual_cf.simple_quals.begin(),
233 qual_cf.simple_quals.end());
234 ra_exe_unit.
quals.insert(
235 ra_exe_unit.
quals.end(), qual_cf.quals.begin(), qual_cf.quals.end());
240 const std::vector<InputTableInfo>& query_infos,
241 const size_t level_idx,
242 const std::string& fail_reason) {
246 if (level_idx + 1 != ra_exe_unit.
join_quals.size()) {
247 throw std::runtime_error(
248 "Hash join failed, reason(s): " + fail_reason +
249 " | Cannot fall back to loop join for intermediate join quals");
252 throw std::runtime_error(
253 "Hash join failed, reason(s): " + fail_reason +
254 " | Cannot fall back to loop join for non-trivial inner table size");
264 const std::vector<InputTableInfo>& query_infos,
268 std::vector<JoinLoop> join_loops;
269 for (
size_t level_idx = 0, current_hash_table_idx = 0;
272 const auto& current_level_join_conditions = ra_exe_unit.
join_quals[level_idx];
273 std::vector<std::string> fail_reasons;
274 const auto build_cur_level_hash_table = [&]() {
275 if (current_level_join_conditions.quals.size() > 1) {
276 const auto first_qual = *current_level_join_conditions.quals.begin();
279 if (qual_bin_oper && qual_bin_oper->is_overlaps_oper() &&
283 return buildCurrentLevelHashTable(
284 join_condition, ra_exe_unit, co, query_infos, column_cache, fail_reasons);
287 return buildCurrentLevelHashTable(current_level_join_conditions,
294 const auto current_level_hash_table = build_cur_level_hash_table();
295 const auto found_outer_join_matches_cb =
296 [
this, level_idx](llvm::Value* found_outer_join_matches) {
300 found_outer_join_matches;
302 const auto is_deleted_cb = buildIsDeletedCb(ra_exe_unit, level_idx, co);
303 const auto outer_join_condition_multi_quals_cb =
304 [
this, level_idx, &co, ¤t_level_join_conditions](
305 const std::vector<llvm::Value*>& prev_iters) {
310 addJoinLoopIterator(prev_iters, level_idx + 1);
316 if (current_level_join_conditions.quals.size() >= 2) {
317 auto qual_it = std::next(current_level_join_conditions.quals.begin(), 1);
318 for (; qual_it != current_level_join_conditions.quals.end();
319 std::advance(qual_it, 1)) {
323 code_generator.
codegen(qual_it->get(),
true, co).front()));
326 return left_join_cond;
328 if (current_level_hash_table) {
330 join_loops.emplace_back(
332 current_level_join_conditions.type,
334 [
this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
335 const std::vector<llvm::Value*>& prev_iters) {
336 addJoinLoopIterator(prev_iters, level_idx);
339 current_level_hash_table->codegenSlot(co, current_hash_table_idx);
344 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
348 join_loops.emplace_back(
350 current_level_join_conditions.type,
352 [
this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
353 const std::vector<llvm::Value*>& prev_iters) {
354 addJoinLoopIterator(prev_iters, level_idx);
356 const auto matching_set = current_level_hash_table->codegenMatchingSet(
357 co, current_hash_table_idx);
359 domain.element_count = matching_set.count;
364 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
365 outer_join_condition_multi_quals_cb)
368 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
372 ++current_hash_table_idx;
374 const auto fail_reasons_str = current_level_join_conditions.quals.empty()
375 ?
"No equijoin expression found"
378 ra_exe_unit, eo, query_infos, level_idx, fail_reasons_str);
381 VLOG(1) <<
"Unable to build hash table, falling back to loop join: "
383 const auto outer_join_condition_cb =
384 [
this, level_idx, &co, ¤t_level_join_conditions](
385 const std::vector<llvm::Value*>& prev_iters) {
390 addJoinLoopIterator(prev_iters, level_idx + 1);
393 for (
auto expr : current_level_join_conditions.quals) {
396 code_generator.toBool(
397 code_generator.codegen(expr.get(),
true, co).front()));
399 return left_join_cond;
401 join_loops.emplace_back(
403 current_level_join_conditions.type,
405 [
this, level_idx](
const std::vector<llvm::Value*>& prev_iters) {
406 addJoinLoopIterator(prev_iters, level_idx);
412 "num_rows_per_scan");
417 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
418 outer_join_condition_cb)
422 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
430 std::function<llvm::Value*(const std::vector<llvm::Value*>&, llvm::Value*)>
432 const size_t level_idx,
439 const auto input_desc = ra_exe_unit.
input_descs[level_idx + 1];
448 CHECK(deleted_cd->columnType.is_boolean());
449 const auto deleted_expr = makeExpr<Analyzer::ColumnVar>(deleted_cd->columnType,
450 input_desc.getTableId(),
451 deleted_cd->columnId,
452 input_desc.getNestLevel());
453 return [
this, deleted_expr, level_idx, &co](
const std::vector<llvm::Value*>& prev_iters,
454 llvm::Value* have_more_inner_rows) {
455 const auto matching_row_index = addJoinLoopIterator(prev_iters, level_idx + 1);
459 llvm::Value* is_valid_it{
nullptr};
460 if (have_more_inner_rows) {
461 is_valid_it = have_more_inner_rows;
464 llvm::ICmpInst::ICMP_SGE, matching_row_index,
cgen_state_->
llInt<int64_t>(0));
466 const auto it_valid_bb = llvm::BasicBlock::Create(
468 const auto it_not_valid_bb = llvm::BasicBlock::Create(
471 const auto row_is_deleted_bb = llvm::BasicBlock::Create(
475 const auto row_is_deleted = code_generator.
toBool(
476 code_generator.
codegen(deleted_expr.get(),
true, co).front());
482 auto row_is_deleted_or_default =
484 row_is_deleted_or_default->addIncoming(row_is_deleted, it_valid_bb);
485 row_is_deleted_or_default->addIncoming(row_is_deleted_default, it_not_valid_bb);
486 return row_is_deleted_or_default;
494 const std::vector<InputTableInfo>& query_infos,
496 std::vector<std::string>& fail_reasons) {
499 current_level_join_conditions.
quals.size() > 1) {
500 fail_reasons.emplace_back(
"No equijoin expression found for outer join");
503 std::shared_ptr<HashJoin> current_level_hash_table;
504 for (
const auto& join_qual : current_level_join_conditions.
quals) {
506 if (!qual_bin_oper || !
IS_EQUIVALENCE(qual_bin_oper->get_optype())) {
507 fail_reasons.emplace_back(
"No equijoin expression found");
514 if (!current_level_hash_table) {
515 hash_table_or_error = buildHashTableForQualifier(
522 current_level_hash_table = hash_table_or_error.
hash_table;
528 fail_reasons.push_back(hash_table_or_error.
fail_reason);
534 return current_level_hash_table;
549 for (
auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
551 for (
auto op_it = instr_it->value_op_begin(); op_it != instr_it->value_op_end();
553 llvm::Value* v = *op_it;
557 if (llvm::dyn_cast<const llvm::CallInst>(instr_it) &&
558 op_it == instr_it->value_op_end() - 1) {
562 if (
auto* instr = llvm::dyn_cast<llvm::Instruction>(v);
563 instr && instr->getParent() &&
567 }
else if (
auto* argum = llvm::dyn_cast<llvm::Argument>(v);
578 std::vector<llvm::Type*> filter_func_arg_types;
581 filter_func_arg_types.push_back(arg->getType());
583 auto ft = llvm::FunctionType::get(
586 auto filter_func2 = llvm::Function::Create(ft,
587 llvm::Function::ExternalLinkage,
593 for (llvm::Function::arg_iterator I = filter_func2->arg_begin(),
594 E = filter_func2->arg_end();
597 arg_it->second = &*I;
598 if (arg_it->first->hasName()) {
599 I->setName(arg_it->first->getName());
608 filter_func2->getBasicBlockList().splice(
620 for (
auto instr_it = bb_it->begin(); instr_it != bb_it->end(); ++instr_it) {
622 for (
auto op_it = instr_it->op_begin(); op_it != instr_it->op_end(); ++op_it, ++i) {
623 llvm::Value* v = op_it->get();
627 llvm::Use* use = &*op_it;
628 use->set(arg_it->second);
636 const size_t level_idx) {
645 CHECK(!prev_iters.empty());
646 llvm::Value* matching_row_index = prev_iters.back();
650 return matching_row_index;
656 llvm::Function* query_func,
657 llvm::BasicBlock* entry_bb,
675 &group_by_and_aggregate,
677 &ra_exe_unit](
const std::vector<llvm::Value*>& prev_iters) {
679 addJoinLoopIterator(prev_iters, join_loops.size());
681 const auto loop_body_bb = llvm::BasicBlock::Create(
682 builder.getContext(),
"loop_body", builder.GetInsertBlock()->getParent());
683 builder.SetInsertPoint(loop_body_bb);
684 const bool can_return_error =
685 compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
688 createErrorCheckControlFlow(query_func,
696 code_generator.
posArg(
nullptr),
705 const size_t col_width,
707 const bool translate_null_val,
708 const int64_t translated_null_val,
710 std::stack<llvm::BasicBlock*>& array_loops,
711 const bool thread_mem_shared) {
713 CHECK_GE(col_width,
sizeof(int32_t));
715 auto group_key = code_generator.
codegen(group_by_col,
true, co).front();
716 auto key_to_cache = group_key;
717 if (dynamic_cast<Analyzer::UOper*>(group_by_col) &&
718 static_cast<Analyzer::UOper*>(group_by_col)->get_optype() ==
kUNNEST) {
723 preheader->getNextNode());
727 CHECK(array_idx_ptr);
729 const auto arr_expr =
static_cast<Analyzer::UOper*
>(group_by_col)->get_operand();
731 CHECK(array_ti.is_array());
732 const auto& elem_ti = array_ti.get_elem_type();
734 (array_ti.get_size() > 0)
740 code_generator.
posArg(arr_expr),
747 llvm::ICmpInst::ICMP_SLT, array_idx, array_len);
748 auto array_loop_body = llvm::BasicBlock::Create(
753 array_loops.empty() ? diamond_codegen.
orig_cond_false_ : array_loops.top());
759 if (array_ti.get_size() < 0) {
760 if (array_ti.get_notnull()) {
761 array_at_fname =
"notnull_" + array_at_fname;
763 array_at_fname =
"varlen_" + array_at_fname;
765 const auto ar_ret_ty =
767 ? (elem_ti.get_type() ==
kDOUBLE
774 {group_key, code_generator.
posArg(arr_expr), array_idx});
776 elem_ti, isArchMaxwell(co.
device_type), thread_mem_shared)) {
777 key_to_cache = spillDoubleElement(group_key, ar_ret_ty);
779 key_to_cache = group_key;
781 CHECK(array_loop_head);
782 array_loops.push(array_loop_head);
785 llvm::Value* orig_group_key{
nullptr};
786 if (translate_null_val) {
787 const std::string translator_func_name(
788 col_width ==
sizeof(int32_t) ?
"translate_null_key_i32_" :
"translate_null_key_");
789 const auto& ti = group_by_col->get_type_info();
791 orig_group_key = group_key;
795 static_cast<llvm::Value*
>(
797 static_cast<llvm::Value*>(llvm::ConstantInt::get(
803 if (orig_group_key) {
808 return {group_key, orig_group_key};
813 llvm::Value* nullable_lv,
815 const std::string&
name)
816 : cgen_state(cgen_state), name(name) {
820 null_check = std::make_unique<GroupByAndAggregate::DiamondCodegen>(
822 ? cgen_state->
ir_builder_.CreateFCmp(llvm::FCmpInst::FCMP_OEQ,
825 : cgen_state->
ir_builder_.CreateICmp(llvm::ICmpInst::ICMP_EQ,
840 cgen_state->
ir_builder_.SetInsertPoint(null_check->cond_true_);
842 cgen_state->
ir_builder_.SetInsertPoint(null_check->cond_false_);
846 llvm::Value* notnull_lv) {
849 cgen_state->ir_builder_.CreateBr(nullcheck_bb);
851 CHECK_EQ(null_lv->getType(), notnull_lv->getType());
853 cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
855 cgen_state->ir_builder_.CreatePHI(null_lv->getType(), 2,
name +
"_value");
856 nullcheck_value->addIncoming(notnull_lv, null_check->cond_false_);
857 nullcheck_value->addIncoming(null_lv, null_check->cond_true_);
859 null_check.reset(
nullptr);
860 cgen_state->ir_builder_.SetInsertPoint(nullcheck_bb);
861 return nullcheck_value;
std::shared_ptr< HashJoin > buildCurrentLevelHashTable(const JoinCondition ¤t_level_join_conditions, RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache, std::vector< std::string > &fail_reasons)
llvm::Value * castToTypeIn(llvm::Value *val, const size_t bit_width)
NullCheckCodegen(CgenState *cgen_state, Executor *executor, llvm::Value *nullable_lv, const SQLTypeInfo &nullable_ti, const std::string &name="")
std::vector< llvm::Value * > outer_join_match_found_per_level_
llvm::BasicBlock * nullcheck_bb
bool is_trivial_loop_join(const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
void codegenJoinLoops(const std::vector< JoinLoop > &join_loops, const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, llvm::Function *query_func, llvm::BasicBlock *entry_bb, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const ExecutionOptions &eo)
llvm::Value * values_buffer
#define IS_EQUIVALENCE(X)
llvm::Value * codegenArith(const Analyzer::BinOper *, const CompilationOptions &)
llvm::Value * emitExternalCall(const std::string &fname, llvm::Type *ret_type, const std::vector< llvm::Value * > args, const std::vector< llvm::Attribute::AttrKind > &fnattrs={}, const bool has_struct_return=false)
llvm::IRBuilder ir_builder_
llvm::Value * posArg(const Analyzer::Expr *) const
std::vector< InputDescriptor > input_descs
bool need_patch_unnest_double(const SQLTypeInfo &ti, const bool is_maxwell, const bool mem_shared)
llvm::ConstantInt * llBool(const bool v) const
virtual std::vector< llvm::Value * > codegenColumn(const Analyzer::ColumnVar *, const bool fetch_column, const CompilationOptions &)
InsertionOrderedMap filter_func_args_
llvm::Value * codegenArrayAt(const Analyzer::BinOper *, const CompilationOptions &)
QualsConjunctiveForm qual_to_conjunctive_form(const std::shared_ptr< Analyzer::Expr > qual_expr)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
std::vector< llvm::Value * > codegenGeoBinOper(const Analyzer::GeoBinOper *, const CompilationOptions &)
std::shared_ptr< HashJoin > hash_table
bool filter_on_deleted_column
llvm::Function * row_func_
llvm::Value * codegenIsNull(const Analyzer::UOper *, const CompilationOptions &)
SQLOps get_optype() const
std::vector< llvm::Value * > group_by_expr_cache_
llvm::LLVMContext & context_
llvm::Function * current_func_
llvm::Value * get_arg_by_name(llvm::Function *func, const std::string &name)
std::vector< llvm::Value * > v_
#define INJECT_TIMER(DESC)
const bool with_dynamic_watchdog
const JoinQualsPerNestingLevel join_quals
std::vector< llvm::Value * > codegenGeoUOper(const Analyzer::GeoUOper *, const CompilationOptions &)
llvm::ConstantInt * inlineIntNull(const SQLTypeInfo &)
void setFalseTarget(llvm::BasicBlock *cond_false)
llvm::Value * codegenFunctionOper(const Analyzer::FunctionOper *, const CompilationOptions &)
static llvm::BasicBlock * codegen(const std::vector< JoinLoop > &join_loops, const std::function< llvm::BasicBlock *(const std::vector< llvm::Value * > &)> &body_codegen, llvm::Value *outer_iter, llvm::BasicBlock *exit_bb, CgenState *cgen_state)
void add_qualifier_to_execution_unit(RelAlgExecutionUnit &ra_exe_unit, const std::shared_ptr< Analyzer::Expr > &qual)
std::unordered_map< int, llvm::Value * > scan_idx_to_hash_pos_
std::vector< llvm::Value * > codegenArrayExpr(const Analyzer::ArrayExpr *, const CompilationOptions &)
const SQLTypeInfo & get_type_info() const
llvm::Value * codegenUMinus(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
llvm::Value * slot_lookup_result
ExecutorDeviceType device_type
std::unordered_map< int, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
const std::vector< InputTableInfo > & query_infos_
llvm::Function * filter_func_
std::unique_ptr< GroupByAndAggregate::DiamondCodegen > null_check
const ColumnDescriptor * getDeletedColForTable(const TableId table_id)
const Expr * get_arg() const
auto find(llvm::Value *key)
Expression class for the LOWER (lowercase) string function. The "arg" constructor parameter must be a...
llvm::Value * toBool(llvm::Value *)
llvm::Value * codegenFunctionOperWithCustomTypeHandling(const Analyzer::FunctionOperWithCustomTypeHandling *, const CompilationOptions &)
llvm::Value * codegenCmp(const Analyzer::BinOper *, const CompilationOptions &)
llvm::BasicBlock * orig_cond_false_
std::list< std::shared_ptr< Analyzer::Expr > > quals
const bool allow_loop_joins
llvm::ConstantInt * llInt(const T v) const
llvm::Value * codegenUnnest(const Analyzer::UOper *, const CompilationOptions &)
llvm::Value * addJoinLoopIterator(const std::vector< llvm::Value * > &prev_iters, const size_t level_idx)
std::list< std::shared_ptr< Analyzer::Expr > > quals
llvm::Value * finalize(llvm::Value *null_lv, llvm::Value *notnull_lv)
llvm::Value * codegenLogical(const Analyzer::BinOper *, const CompilationOptions &)
void check_if_loop_join_is_allowed(RelAlgExecutionUnit &ra_exe_unit, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, const size_t level_idx, const std::string &fail_reason)
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
llvm::ConstantInt * ll_bool(const bool v, llvm::LLVMContext &context)
llvm::Value * codegenCast(const Analyzer::UOper *, const CompilationOptions &)
GroupColLLVMValue groupByColumnCodegen(Analyzer::Expr *group_by_col, const size_t col_width, const CompilationOptions &, const bool translate_null_val, const int64_t translated_null_val, GroupByAndAggregate::DiamondCodegen &, std::stack< llvm::BasicBlock * > &, const bool thread_mem_shared)
uint32_t log2_bytes(const uint32_t bytes)
std::string numeric_type_name(const SQLTypeInfo &ti)
void redeclareFilterFunction()
std::vector< JoinLoop > buildJoinLoops(RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache)
std::function< llvm::Value *(const std::vector< llvm::Value * > &, llvm::Value *)> buildIsDeletedCb(const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const CompilationOptions &co)
const bool allow_runtime_query_interrupt
SQLOps get_optype() const
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals
llvm::ConstantFP * inlineFpNull(const SQLTypeInfo &)
Executor * executor() const