27 const auto window_func_context =
31 switch (window_func->getKind()) {
36 return code_generator.codegenWindowPosition(window_func_context,
37 code_generator.posArg(
nullptr));
40 return cgen_state_->emitCall(
"percent_window_func",
41 {cgen_state_->llInt(reinterpret_cast<const int64_t>(
42 window_func_context->output())),
43 code_generator.posArg(
nullptr)});
51 const auto&
args = window_func->getArgs();
53 const auto arg_lvs = code_generator.codegen(
args.front().get(),
true, co);
55 return arg_lvs.front();
65 return codegenWindowFunctionAggregate(&code_generator, co);
73 return codegenWindowNavigationFunctionOnFrame(co);
75 LOG(
FATAL) <<
"Invalid window function kind";
101 agg_name =
"agg_count";
105 agg_name =
"agg_count_if";
109 agg_name =
"agg_sum_if";
113 LOG(
FATAL) <<
"Invalid window function kind";
116 switch (window_func_ti.
get_type()) {
118 agg_name +=
"_float";
122 agg_name +=
"_double";
136 ?
args.front()->get_type_info()
149 }
else if (size == 2) {
151 }
else if (size == 4) {
158 if (col_ti.
is_fp()) {
166 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
169 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
173 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
175 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
177 return cgen_state->
llInt((int64_t)inline_int_null_value<int64_t>());
183 return cgen_state->
llInt((int8_t)inline_int_null_value<int8_t>());
185 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
187 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
198 return cgen_state->
llInt((int64_t)inline_int_null_value<int64_t>());
208 if (col_ti.
is_fp()) {
215 llvm::Value* ret_val{
nullptr};
223 if (ret_val->getType()->getIntegerBitWidth() > ret_val_col_size_in_bytes) {
224 return cgen_state->
castToTypeIn(ret_val, ret_val_col_size_in_bytes);
235 const auto window_func_context =
237 const auto window_func = window_func_context->getWindowFunction();
240 arg_ti.get_type() ==
kFLOAT
241 ? llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0)
242 : llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
243 const auto aggregate_state_i64 = cgen_state_->llInt(
244 reinterpret_cast<const int64_t>(window_func_context->aggregateState()));
250 aggregate_state_type,
258 auto [reset_state_false_bb, aggregate_state] =
259 codegenWindowResetStateControlFlow(code_generator, co);
260 llvm::Value* aggregate_state_count =
nullptr;
261 const auto window_func_context =
263 const auto window_func = window_func_context->getWindowFunction();
265 const auto aggregate_state_count_i64 = cgen_state_->llInt(
266 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
267 const auto pi64_type =
268 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
273 aggregate_state_count_i64,
278 codegenWindowFunctionStateInit(code_generator, co, aggregate_state);
280 const auto count_zero = cgen_state_->llInt(int64_t(0));
281 cgen_state_->emitCall(
"agg_id", {aggregate_state_count, count_zero});
283 cgen_state_->ir_builder_.CreateBr(reset_state_false_bb);
284 cgen_state_->ir_builder_.SetInsertPoint(reset_state_false_bb);
286 return codegenWindowFunctionAggregateCalls(aggregate_state, co);
293 const auto window_func_context =
295 auto aggregate_state = aggregateWindowStatePtr(code_generator, co);
296 const auto bitset = cgen_state_->llInt(
297 reinterpret_cast<const int64_t>(window_func_context->partitionStart()));
298 const auto bitset_lv =
304 llvm::PointerType::get(
get_int_type(8, cgen_state_->context_), 0),
307 const auto min_val = cgen_state_->llInt(int64_t(0));
308 const auto max_val = cgen_state_->llInt(window_func_context->elementCount() - 1);
309 const auto null_val = cgen_state_->llInt(inline_int_null_value<int64_t>());
310 const auto null_bool_val = cgen_state_->llInt<int8_t>(inline_int_null_value<int8_t>());
311 const auto reset_state =
312 code_generator->
toBool(cgen_state_->emitCall(
"bit_is_set",
314 code_generator->
posArg(
nullptr),
319 const auto reset_state_true_bb = llvm::BasicBlock::Create(
320 cgen_state_->context_,
"reset_state.true", cgen_state_->current_func_);
321 const auto reset_state_false_bb = llvm::BasicBlock::Create(
322 cgen_state_->context_,
"reset_state.false", cgen_state_->current_func_);
323 cgen_state_->ir_builder_.CreateCondBr(
324 reset_state, reset_state_true_bb, reset_state_false_bb);
325 cgen_state_->ir_builder_.SetInsertPoint(reset_state_true_bb);
326 return std::make_pair(reset_state_false_bb, aggregate_state);
331 llvm::Value* aggregate_state) {
333 const auto window_func_context =
335 const auto window_func = window_func_context->getWindowFunction();
337 const auto window_func_null_val =
338 window_func_ti.is_fp()
339 ? cgen_state_->inlineFpNull(window_func_ti)
340 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
341 llvm::Value* window_func_init_val;
342 const auto window_func_kind = window_func_context->getWindowFunction()->getKind();
345 switch (window_func_ti.get_type()) {
347 window_func_init_val = cgen_state_->llFp(
float(0));
351 window_func_init_val = cgen_state_->llFp(
double(0));
355 window_func_init_val = cgen_state_->llInt(int64_t(0));
360 window_func_init_val = window_func_null_val;
362 const auto pi32_type =
363 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
364 switch (window_func_ti.get_type()) {
366 cgen_state_->emitCall(
"agg_id_double", {aggregate_state, window_func_init_val});
371 cgen_state_->ir_builder_.CreateBitCast(aggregate_state, pi32_type);
372 cgen_state_->emitCall(
"agg_id_float", {aggregate_state, window_func_init_val});
376 cgen_state_->emitCall(
"agg_id", {aggregate_state, window_func_init_val});
385 const auto window_func_context =
387 const auto window_func = window_func_context->getWindowFunction();
388 const auto window_func_kind = window_func->getKind();
389 const auto&
args = window_func->getArgs();
393 const auto target_col_ti =
args.front()->get_type_info();
394 const auto target_col_size = target_col_ti.get_size();
395 const auto target_col_type_name =
398 window_func->get_type_info().get_size(), window_func->get_type_info().is_fp());
403 auto logical_null_val_lv =
405 auto target_col_null_val_lv =
407 if (window_func_context->elementCount() == 0) {
409 return target_col_null_val_lv;
412 auto current_row_pos_lv = code_generator.posArg(
nullptr);
413 auto partition_index_lv = codegenCurrentPartitionIndex(
414 window_func_context, &code_generator, co, current_row_pos_lv);
417 size_t target_col_size_in_byte = target_col_size * 8;
419 target_col_ti.is_fp()
420 ?
get_fp_type(target_col_size_in_byte, cgen_state_->context_)
421 :
get_int_type(target_col_size_in_byte, cgen_state_->context_);
422 auto col_buf_type = llvm::PointerType::get(col_buf_ptr_type, 0);
423 auto target_col_buf_ptr_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
424 window_func_context->getColumnBufferForWindowFunctionExpressions().front()));
429 target_col_buf_ptr_lv,
435 auto partition_buf_ptrs = codegenLoadPartitionBuffers(
436 window_func_context, &code_generator, co, partition_index_lv);
439 const auto order_key_buf_ti =
440 window_func_context->getOrderKeyColumnBufferTypes().front();
441 auto const ordering_spec = window_func->getCollation().front();
442 auto order_key_col_null_val_lv =
446 auto [order_col_type_name, order_key_buf_ptr_lv] =
447 codegenLoadOrderKeyBufPtr(window_func_context, &code_generator, co);
450 auto [null_start_pos_lv, null_end_pos_lv] =
451 codegenFrameNullRange(window_func_context, &code_generator, co, partition_index_lv);
454 std::string row_idx_on_frame_func =
"compute_";
455 row_idx_on_frame_func += order_col_type_name;
456 row_idx_on_frame_func += ordering_spec.is_desc ?
"_greater_equal" :
"_less_equal";
457 row_idx_on_frame_func +=
"_current_row_idx_in_frame";
458 auto int64_t_one_val_lv = cgen_state_->llInt((int64_t)1);
459 auto nulls_first_lv = cgen_state_->llBool(ordering_spec.nulls_first);
460 auto cur_row_idx_in_frame_lv =
461 cgen_state_->emitCall(row_idx_on_frame_func,
462 {partition_buf_ptrs.num_elem_current_partition_lv,
464 order_key_buf_ptr_lv,
465 partition_buf_ptrs.target_partition_rowid_ptr_lv,
466 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
467 order_key_col_null_val_lv,
472 if (window_func->isMissingValueFillingFunction()) {
480 std::string func_name =
"fill_" + target_col_type_name +
"_missing_value";
482 llvm::Value* forward_fill_lv =
484 return cgen_state_->emitCall(func_name,
485 {cur_row_idx_in_frame_lv,
486 target_col_null_val_lv,
488 partition_buf_ptrs.num_elem_current_partition_lv,
489 partition_buf_ptrs.target_partition_rowid_ptr_lv,
490 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
495 auto [frame_start_bound_expr_lv, frame_end_bound_expr_lv] =
496 codegenFrameBoundRange(window_func, code_generator, co);
499 auto const int64_t_zero_val_lv = cgen_state_->llInt((int64_t)0);
501 frame_start_bound_expr_lv,
502 frame_end_bound_expr_lv,
503 window_func->hasRangeModeFraming() ? current_row_pos_lv : cur_row_idx_in_frame_lv,
505 window_func->hasRangeModeFraming()
506 ? int64_t_zero_val_lv
510 partition_buf_ptrs.num_elem_current_partition_lv,
511 order_key_buf_ptr_lv,
513 partition_buf_ptrs.target_partition_rowid_ptr_lv,
514 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
518 auto [frame_start_bound_lv, frame_end_bound_lv] =
519 codegenWindowFrameBounds(window_func_context,
520 window_func->getFrameStartBound(),
521 window_func->getFrameEndBound(),
522 order_key_col_null_val_lv,
527 llvm::Value* modified_cur_row_idx_in_frame_lv{
nullptr};
528 llvm::Value* offset_lv{
nullptr};
529 switch (window_func_kind) {
531 offset_lv = cgen_state_->castToTypeIn(
532 code_generator.codegen(
args[1].get(),
true, co)[0], 64);
533 modified_cur_row_idx_in_frame_lv =
534 cgen_state_->ir_builder_.CreateSub(cur_row_idx_in_frame_lv, offset_lv);
537 offset_lv = cgen_state_->castToTypeIn(
538 code_generator.codegen(
args[1].get(),
true, co)[0], 64);
539 modified_cur_row_idx_in_frame_lv =
540 cgen_state_->ir_builder_.CreateAdd(cur_row_idx_in_frame_lv, offset_lv);
543 modified_cur_row_idx_in_frame_lv = frame_start_bound_lv;
546 modified_cur_row_idx_in_frame_lv = frame_end_bound_lv;
549 offset_lv = cgen_state_->castToTypeIn(
550 code_generator.codegen(
args[1].get(),
true, co)[0], 64);
551 auto candidate_offset_lv =
552 cgen_state_->ir_builder_.CreateAdd(frame_start_bound_lv, offset_lv);
553 auto out_of_frame_bound_lv =
554 cgen_state_->ir_builder_.CreateICmpSGT(candidate_offset_lv, frame_end_bound_lv);
555 modified_cur_row_idx_in_frame_lv = cgen_state_->ir_builder_.CreateSelect(
556 out_of_frame_bound_lv, cgen_state_->llInt((int64_t)-1), candidate_offset_lv);
560 UNREACHABLE() <<
"Unsupported window function to navigate a window frame.";
562 CHECK(modified_cur_row_idx_in_frame_lv);
565 std::string target_func_name =
"get_";
566 target_func_name += target_col_type_name +
"_value_";
567 target_func_name += target_col_logical_type_name +
"_type_";
568 target_func_name +=
"in_frame";
570 cgen_state_->emitCall(target_func_name,
571 {modified_cur_row_idx_in_frame_lv,
572 frame_start_bound_lv,
575 partition_buf_ptrs.target_partition_rowid_ptr_lv,
576 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
578 target_col_null_val_lv});
580 res_lv = cgen_state_->emitCall(
582 {res_lv, logical_null_val_lv, cgen_state_->llInt((int64_t)
kSecsPerDay)});
596 const auto order_col_ti = window_func->
getOrderKeys().front()->get_type_info();
597 auto encode_date_col_val = [&order_col_ti,
this](llvm::Value* bound_expr_lv) {
598 if (order_col_ti.get_comp_param() == 16) {
599 return cgen_state_->emitCall(
600 "fixed_width_date_encode_noinline",
606 return cgen_state_->emitCall(
"fixed_width_date_encode_noinline",
612 llvm::Value* bound_expr_lv{
nullptr};
613 if (needs_bound_expr_codegen(frame_bound)) {
615 if (
auto dateadd_expr = dynamic_cast<const Analyzer::DateaddExpr*>(bound_expr)) {
616 if (dateadd_expr->get_datetime_expr()->get_type_info().is_encoded_timestamp()) {
617 dateadd_expr->set_fixed_encoding_null_val();
620 auto bound_expr_lvs = code_generator.
codegen(bound_expr,
true, co);
621 bound_expr_lv = bound_expr_lvs.front();
624 throw std::runtime_error(
625 "Range mode with date type ordering column is not supported yet.");
627 bound_expr_lv = encode_date_col_val(bound_expr_lv);
630 bound_expr_lv = cgen_state_->castToTypeIn(bound_expr_lv, 64);
633 bound_expr_lv = cgen_state_->llInt((int64_t)-1);
635 CHECK(bound_expr_lv);
636 return bound_expr_lv;
641 bool for_window_frame_naviation,
643 bool is_timestamp_type_frame,
644 llvm::Value* order_key_null_val,
647 auto adjust_frame_end_bound = [&](llvm::Value* target_bound_lv) {
651 CHECK(for_start_bound) <<
"frame end cannot be UNBOUNDED PRECEDING";
654 CHECK(!for_start_bound) <<
"frame start cannot be UNBOUNDED FOLLOWING";
656 return for_window_frame_naviation
660 std::vector<llvm::Value*> func_args;
661 std::string op_name =
663 if (!for_range_mode) {
664 llvm::Value* current_row_bound_expr_lv{
nullptr};
665 if (for_window_frame_naviation) {
670 current_row_bound_expr_lv =
673 current_row_bound_expr_lv =
680 if (for_start_bound) {
681 return cgen_state_->ir_builder_.CreateSelect(
682 cgen_state_->ir_builder_.CreateICmpSLT(current_row_bound_expr_lv,
685 current_row_bound_expr_lv);
687 return cgen_state_->ir_builder_.CreateSelect(
688 cgen_state_->ir_builder_.CreateICmpSGE(current_row_bound_expr_lv,
691 current_row_bound_expr_lv);
694 std::string func_class = for_start_bound ?
"start" :
"end";
695 auto const func_name =
"compute_row_mode_" + func_class +
"_index_" + op_name;
696 func_args = prepareRowModeFuncArgs(for_start_bound, bound_type, args);
697 current_row_bound_expr_lv = cgen_state_->emitCall(func_name, func_args);
699 return current_row_bound_expr_lv;
701 std::string func_class = for_start_bound ?
"lower" :
"upper";
702 auto const func_name = getFramingFuncName(
707 func_args = prepareRangeModeFuncArgs(
708 for_start_bound, frame_bound, is_timestamp_type_frame, order_key_null_val, args);
709 auto frame_bound_lv = cgen_state_->emitCall(func_name, func_args);
710 if (!for_start_bound && for_window_frame_naviation) {
712 frame_bound_lv = cgen_state_->ir_builder_.CreateSelect(
713 cgen_state_->ir_builder_.CreateICmpSGE(frame_bound_lv,
718 return frame_bound_lv;
725 return window_func->
getOrderKeys().front()->get_type_info();
729 const auto order_key_size = getFirstOrderColTypeInfo(window_func_context).get_size();
730 return order_key_size;
735 auto const order_key_size = getOrderKeySize(window_func_context);
736 auto const order_key_ptr =
738 CHECK(order_key_ptr);
740 order_key_ptr->get_type_info().is_fp());
747 llvm::Value* current_col_value_ptr_lv{
nullptr};
748 const auto order_key_size_in_byte = getOrderKeySize(window_func_context) * 8;
749 auto const order_key_ptr =
751 CHECK(order_key_ptr);
752 auto const order_col_ti = order_key_ptr->get_type_info();
753 auto const order_col_llvm_type =
754 order_col_ti.is_fp() ?
get_fp_type(order_key_size_in_byte, cgen_state_->context_)
755 :
get_int_type(order_key_size_in_byte, cgen_state_->context_);
759 current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
762 current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
765 return cgen_state_->ir_builder_.CreateLoad(
766 current_col_value_ptr_lv->getType()->getPointerElementType(),
767 current_col_value_ptr_lv,
768 "current_col_value");
775 llvm::Value* current_row_pos_lv) {
776 const auto pi64_type =
777 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
778 const auto pi32_type =
779 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
780 auto row_pos_lv = current_row_pos_lv;
798 auto*
const hash_slot_idx_ptr =
800 auto hash_slot_idx_buf_lv =
801 cgen_state_->llInt(reinterpret_cast<int64_t>(hash_slot_idx_ptr));
806 hash_slot_idx_buf_lv,
810 auto hash_slot_idx_load_lv = cgen_state_->ir_builder_.CreateGEP(
811 hash_slot_idx_ptr_lv->getType()->getPointerElementType(),
812 hash_slot_idx_ptr_lv,
814 row_pos_lv = cgen_state_->castToTypeIn(
815 cgen_state_->ir_builder_.CreateLoad(
816 hash_slot_idx_load_lv->getType()->getPointerElementType(),
817 hash_slot_idx_load_lv,
818 "cur_row_hash_slot_idx"),
821 auto partition_count_lv = cgen_state_->llInt(window_func_context->
partitionCount());
822 auto partition_num_count_buf_lv = cgen_state_->llInt(
828 partition_num_count_buf_lv,
832 return cgen_state_->emitCall(
833 "compute_int64_t_lower_bound",
834 {partition_count_lv, row_pos_lv, partition_num_count_ptr_lv});
838 const std::string& order_col_type,
839 const std::string& op_type,
840 bool for_timestamp_type)
const {
841 auto target_val_type = for_timestamp_type ?
"int64_t" : order_col_type;
842 auto null_type = for_timestamp_type ?
"int64_t" : order_col_type;
843 return "range_mode_" + target_val_type +
"_" + order_col_type +
"_" + null_type +
"_" +
844 op_type +
"_frame_" + bound_type +
"_bound";
848 bool for_start_bound,
866 bool for_start_bound,
868 bool is_timestamp_type_frame,
869 llvm::Value* order_key_null_val,
871 llvm::Value* bound_expr_lv =
873 llvm::Value* target_val_lv =
877 llvm::Value* frame_bound_val_lv =
898 llvm::Value* partition_index_lv)
const {
899 const auto pi64_type =
900 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
901 const auto null_start_pos_buf = cgen_state_->llInt(
911 const auto null_start_pos_ptr =
912 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
913 null_start_pos_buf_ptr,
915 auto null_start_pos_lv = cgen_state_->ir_builder_.CreateLoad(
916 null_start_pos_ptr->getType()->getPointerElementType(),
919 const auto null_end_pos_buf = cgen_state_->llInt(
929 const auto null_end_pos_ptr = cgen_state_->ir_builder_.CreateGEP(
930 get_int_type(64, cgen_state_->context_), null_end_pos_buf_ptr, partition_index_lv);
931 auto null_end_pos_lv = cgen_state_->ir_builder_.CreateLoad(
932 null_end_pos_ptr->getType()->getPointerElementType(),
935 return std::make_pair(null_start_pos_lv, null_end_pos_lv);
942 auto const order_key_ti =
944 auto const order_key_size = order_key_ti.get_size();
948 size_t order_key_size_in_byte = order_key_size * 8;
949 auto const order_key_type =
950 order_key_ti.is_fp() ?
get_fp_type(order_key_size_in_byte, cgen_state_->context_)
951 :
get_int_type(order_key_size_in_byte, cgen_state_->context_);
952 auto const order_key_buf_type = llvm::PointerType::get(order_key_type, 0);
953 auto const order_key_buf = cgen_state_->llInt(
963 return std::make_pair(order_col_type_name, order_key_buf_ptr_lv);
970 llvm::Value* partition_index_lv)
const {
972 const auto pi64_type =
973 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
974 const auto pi32_type =
975 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
978 auto partition_start_offset_buf_lv = cgen_state_->llInt(
984 partition_start_offset_buf_lv,
990 auto current_partition_start_offset_ptr_lv =
991 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
992 partition_start_offset_ptr_lv,
995 current_partition_start_offset_ptr_lv->getType()->getPointerElementType(),
996 current_partition_start_offset_ptr_lv);
999 const auto partition_rowid_buf_lv =
1000 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->
payload()));
1005 partition_rowid_buf_lv,
1010 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
1011 partition_rowid_ptr_lv,
1015 const auto sorted_rowid_lv = cgen_state_->llInt(
1026 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
1027 sorted_rowid_ptr_lv,
1031 const auto partition_count_buf =
1032 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->
counts()));
1037 partition_count_buf,
1043 const auto num_elem_current_partition_ptr =
1044 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
1045 partition_count_buf_ptr_lv,
1046 partition_index_lv);
1048 cgen_state_->ir_builder_.CreateLoad(
1049 num_elem_current_partition_ptr->getType()->getPointerElementType(),
1050 num_elem_current_partition_ptr),
1061 auto frame_start_bound_expr_lv =
1062 codegenFrameBoundExpr(window_func, frame_start_bound, code_generator, co);
1063 auto frame_end_bound_expr_lv =
1064 codegenFrameBoundExpr(window_func, frame_end_bound, code_generator, co);
1065 CHECK(frame_start_bound_expr_lv);
1066 CHECK(frame_end_bound_expr_lv);
1067 return std::make_pair(frame_start_bound_expr_lv, frame_end_bound_expr_lv);
1074 llvm::Value* order_key_col_null_val_lv,
1082 if (window_func->hasRangeModeFraming()) {
1084 CHECK(window_func->getOrderKeys().size() == 1UL);
1088 codegenLoadCurrentValueFromColBuf(window_func_context, code_generator, args);
1091 auto get_order_key_null_val = [is_timestamp_type_frame,
1092 &order_key_col_null_val_lv,
1094 return is_timestamp_type_frame && !frame_bound->isCurrentRowBound()
1095 ? cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64)
1096 : order_key_col_null_val_lv;
1098 auto frame_start_bound_lv =
1099 codegenFrameBound(
true,
1100 window_func->hasRangeModeFraming(),
1101 window_func->isFrameNavigateWindowFunction(),
1103 is_timestamp_type_frame,
1104 get_order_key_null_val(frame_start_bound),
1106 auto frame_end_bound_lv =
1107 codegenFrameBound(
false,
1108 window_func->hasRangeModeFraming(),
1109 window_func->isFrameNavigateWindowFunction(),
1111 is_timestamp_type_frame,
1112 get_order_key_null_val(frame_end_bound),
1114 CHECK(frame_start_bound_lv);
1115 CHECK(frame_end_bound_lv);
1116 return std::make_pair(frame_start_bound_lv, frame_end_bound_lv);
1122 const auto window_func_context =
1124 const auto window_func = window_func_context->getWindowFunction();
1126 const auto window_func_null_val =
1127 window_func_ti.is_fp()
1128 ? cgen_state_->inlineFpNull(window_func_ti)
1129 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
1130 if (window_func_context->elementCount() == 0) {
1134 : window_func_null_val;
1136 const auto&
args = window_func->getArgs();
1138 if (window_func_context->needsToBuildAggregateTree()) {
1143 const auto pi64_type =
1144 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1145 const auto ppi64_type = llvm::PointerType::get(
1146 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0), 0);
1148 auto [frame_start_bound_expr_lv, frame_end_bound_expr_lv] =
1149 codegenFrameBoundRange(window_func, code_generator, co);
1152 auto current_row_pos_lv = code_generator.
posArg(
nullptr);
1153 auto partition_index_lv = codegenCurrentPartitionIndex(
1154 window_func_context, &code_generator, co, current_row_pos_lv);
1157 const auto target_col_ti =
args.front()->get_type_info();
1158 const auto target_col_size = target_col_ti.get_size();
1159 const auto col_type_name =
1162 const auto partition_buf_ptrs = codegenLoadPartitionBuffers(
1163 window_func_context, &code_generator, co, partition_index_lv);
1165 auto [order_col_type_name, order_key_buf_ptr_lv] =
1166 codegenLoadOrderKeyBufPtr(window_func_context, &code_generator, co);
1169 const auto order_key_buf_ti =
1170 window_func_context->getOrderKeyColumnBufferTypes().front();
1171 auto const ordering_spec = window_func->getCollation().front();
1172 llvm::Value* order_key_col_null_val_lv{
nullptr};
1173 switch (order_key_buf_ti.get_type()) {
1180 order_key_col_null_val_lv = cgen_state_->llInt((int32_t)null_val);
1185 order_key_col_null_val_lv = cgen_state_->inlineNull(order_key_buf_ti);
1190 auto [null_start_pos_lv, null_end_pos_lv] = codegenFrameNullRange(
1191 window_func_context, &code_generator, co, partition_index_lv);
1192 auto nulls_first_lv = cgen_state_->llBool(ordering_spec.nulls_first);
1195 frame_start_bound_expr_lv,
1196 frame_end_bound_expr_lv,
1200 cgen_state_->llInt((int64_t)0),
1201 cgen_state_->llInt((int64_t)1),
1202 partition_buf_ptrs.num_elem_current_partition_lv,
1203 order_key_buf_ptr_lv,
1205 partition_buf_ptrs.target_partition_rowid_ptr_lv,
1206 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
1210 auto [frame_start_bound_lv, frame_end_bound_lv] =
1211 codegenWindowFrameBounds(window_func_context,
1212 window_func->getFrameStartBound(),
1213 window_func->getFrameEndBound(),
1214 order_key_col_null_val_lv,
1219 llvm::ConstantInt* aggregation_trees_lv{
nullptr};
1220 llvm::Value* invalid_val_lv{
nullptr};
1221 llvm::Value* null_val_lv{
nullptr};
1222 std::string aggregation_tree_search_func_name{
"search_"};
1223 std::string aggregation_tree_getter_func_name{
"get_"};
1227 auto agg_expr_ti =
args.front()->get_type_info();
1228 if (agg_expr_ti.is_fp()) {
1230 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::max());
1232 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::lowest());
1234 invalid_val_lv = cgen_state_->llFp((
double)0);
1237 aggregation_tree_search_func_name +=
"double";
1238 aggregation_tree_getter_func_name +=
"double";
1241 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::max());
1243 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::lowest());
1245 invalid_val_lv = cgen_state_->llInt((int64_t)0);
1247 null_val_lv = cgen_state_->llInt(inline_int_null_value<int64_t>());
1248 aggregation_tree_search_func_name +=
"int64_t";
1249 aggregation_tree_getter_func_name +=
"integer";
1254 aggregation_tree_search_func_name +=
"_derived";
1255 aggregation_tree_getter_func_name +=
"_derived";
1259 if (agg_expr_ti.is_fp()) {
1261 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1262 window_func_context->getDerivedAggregationTreesForDoubleTypeWindowExpr()));
1264 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1265 window_func_context->getAggregationTreesForDoubleTypeWindowExpr()));
1269 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1270 window_func_context->getDerivedAggregationTreesForIntegerTypeWindowExpr()));
1272 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1273 window_func_context->getAggregationTreesForIntegerTypeWindowExpr()));
1277 CHECK(aggregation_trees_lv);
1278 CHECK(invalid_val_lv);
1279 aggregation_tree_search_func_name +=
"_aggregation_tree";
1280 aggregation_tree_getter_func_name +=
"_aggregation_tree";
1287 aggregation_trees_lv,
1291 auto target_aggregation_tree_lv = cgen_state_->emitCall(
1292 aggregation_tree_getter_func_name, {aggregation_trees_ptr, partition_index_lv});
1295 const auto tree_depth_buf = cgen_state_->llInt(
1296 reinterpret_cast<int64_t>(window_func_context->getAggregateTreeDepth()));
1305 const auto current_partition_tree_depth_buf_ptr = cgen_state_->ir_builder_.CreateGEP(
1306 get_int_type(64, cgen_state_->context_), tree_depth_buf_ptr, partition_index_lv);
1307 const auto current_partition_tree_depth_lv = cgen_state_->ir_builder_.CreateLoad(
1308 current_partition_tree_depth_buf_ptr->getType()->getPointerElementType(),
1309 current_partition_tree_depth_buf_ptr);
1312 const auto aggregation_tree_fanout_lv = cgen_state_->llInt(
1313 static_cast<int64_t>(window_func_context->getAggregateTreeFanout()));
1316 const auto agg_type_lv =
1317 cgen_state_->llInt(static_cast<int32_t>(window_func->getKind()));
1322 cgen_state_->emitCall(aggregation_tree_search_func_name,
1323 {target_aggregation_tree_lv,
1324 frame_start_bound_lv,
1326 current_partition_tree_depth_lv,
1327 aggregation_tree_fanout_lv,
1328 cgen_state_->llBool(agg_expr_ti.is_decimal()),
1329 cgen_state_->llInt((int64_t)agg_expr_ti.get_scale()),
1335 std::string null_handler_func_name{
"handle_null_val_"};
1336 std::vector<llvm::Value*> null_handler_args{res_lv, null_val_lv};
1342 if (agg_expr_ti.is_fp()) {
1344 null_handler_func_name +=
"double_double";
1347 null_handler_func_name +=
"double_int64_t";
1349 }
else if (agg_expr_ti.is_fp()) {
1351 null_handler_func_name +=
"double_double";
1354 null_handler_func_name +=
"int64_t_int64_t";
1356 null_handler_func_name +=
"_window_framing_agg";
1360 if (agg_expr_ti.is_fp()) {
1361 null_handler_args.push_back(cgen_state_->llFp((
double)0));
1363 null_handler_args.push_back(cgen_state_->llInt((int64_t)0));
1368 null_handler_args.push_back(cgen_state_->castToTypeIn(window_func_null_val, 64));
1370 res_lv = cgen_state_->emitCall(null_handler_func_name, null_handler_args);
1375 return cgen_state_->ir_builder_.CreateFPToSI(
1378 agg_expr_ti.is_date_in_days()) {
1381 if (date_null_val->getType()->getScalarSizeInBits() != 32) {
1382 date_null_val = cgen_state_->castToTypeIn(date_null_val, 32);
1384 return cgen_state_->emitCall(
"fixed_width_date_decode",
1385 {res_lv, date_null_val, null_val_lv});
1391 std::vector<llvm::Value*> agg_func_args{aggregate_state};
1392 auto modified_window_func_null_val = window_func_null_val;
1397 agg_func_args.push_back(cgen_state_->llInt(int64_t(1)));
1401 arg_target_expr =
args.front().get();
1402 const auto arg_lvs = code_generator.
codegen(arg_target_expr,
true, co);
1403 CHECK_EQ(arg_lvs.size(), size_t(1));
1405 auto crt_val = arg_lvs.front();
1408 !window_func_ti.is_fp()) {
1410 arg_lvs.front(),
args.front()->get_type_info(), window_func_ti,
false);
1412 agg_func_args.push_back(window_func_ti.get_type() ==
kFLOAT
1414 : cgen_state_->castToTypeIn(crt_val, 64));
1416 llvm::Value* cond_lv{
nullptr};
1418 switch (window_func->getKind()) {
1421 modified_window_func_null_val = cgen_state_->castToTypeIn(
1427 if (
args.front()->get_type_info().is_integer()) {
1428 agg_func_args[1] = cgen_state_->castToTypeIn(agg_func_args[1], 64);
1430 modified_window_func_null_val =
1431 cgen_state_->castToTypeIn(window_func_null_val, 64);
1433 auto cond_expr_lv = code_generator.
codegen(
args[1].
get(),
true, co).front();
1435 codegenConditionalAggregateCondValSelector(cond_expr_lv,
kSUM_IF, co);
1441 agg_name +=
"_skip_val";
1442 agg_func_args.push_back(modified_window_func_null_val);
1444 agg_func_args.push_back(cond_lv);
1447 cgen_state_->emitCall(agg_name, agg_func_args);
1449 codegenWindowAvgEpilogue(
1450 &code_generator, co, agg_func_args[1], window_func_null_val);
1452 return codegenAggregateWindowState(&code_generator, co, aggregate_state);
1458 llvm::Value* crt_val,
1459 llvm::Value* window_func_null_val) {
1461 const auto window_func_context =
1463 const auto window_func = window_func_context->getWindowFunction();
1465 const auto pi32_type =
1466 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1467 const auto pi64_type =
1468 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1469 const auto aggregate_state_type =
1470 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1471 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1472 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1477 aggregate_state_count_i64,
1478 aggregate_state_type,
1481 std::string agg_count_func_name =
"agg_count";
1482 switch (window_func_ti.get_type()) {
1484 agg_count_func_name +=
"_float";
1488 agg_count_func_name +=
"_double";
1495 agg_count_func_name +=
"_skip_val";
1496 cgen_state_->emitCall(agg_count_func_name,
1497 {aggregate_state_count, crt_val, window_func_null_val});
1502 llvm::Value* aggregate_state) {
1504 const auto pi32_type =
1505 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1506 const auto pi64_type =
1507 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1508 const auto window_func_context =
1512 const auto aggregate_state_type =
1513 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1515 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1516 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1521 aggregate_state_count_i64,
1522 aggregate_state_type,
1526 switch (window_func_ti.get_type()) {
1528 return cgen_state_->emitCall(
1529 "load_avg_float", {aggregate_state, aggregate_state_count, double_null_lv});
1532 return cgen_state_->emitCall(
1533 "load_avg_double", {aggregate_state, aggregate_state_count, double_null_lv});
1536 return cgen_state_->emitCall(
1539 aggregate_state_count,
1541 cgen_state_->llInt<int32_t>(window_func_ti.get_scale())});
1544 return cgen_state_->emitCall(
1545 "load_avg_int", {aggregate_state, aggregate_state_count, double_null_lv});
1550 return cgen_state_->ir_builder_.CreateLoad(
1551 aggregate_state->getType()->getPointerElementType(), aggregate_state);
1553 switch (window_func_ti.get_type()) {
1555 return cgen_state_->emitCall(
"load_float", {aggregate_state});
1558 return cgen_state_->emitCall(
"load_double", {aggregate_state});
1561 return cgen_state_->ir_builder_.CreateLoad(
1562 aggregate_state->getType()->getPointerElementType(), aggregate_state);
1568 llvm::Value* cond_lv,
1571 llvm::Value* res_cond_lv{
nullptr};
1574 if (cond_lv->getType()->isIntegerTy(1)) {
1577 res_cond_lv = cgen_state_->castToTypeIn(cond_lv, 8);
1579 CHECK(cond_lv->getType()->isIntegerTy(8));
1585 cgen_state_->ir_builder_.CreateICmpEQ(cond_lv, cgen_state_->llInt((int8_t)1));
1586 res_cond_lv = cgen_state_->ir_builder_.CreateSelect(
1587 true_cond_lv, cgen_state_->llInt((int8_t)1), cgen_state_->llInt((int8_t)0));
std::string get_col_type_name_by_size(const size_t size, const bool is_fp)
bool hasRangeModeFraming() const
bool isCurrentRowBound() const
llvm::Value * castToTypeIn(llvm::Value *val, const size_t bit_width)
SqlWindowFunctionKind getKind() const
llvm::Value * num_elem_current_partition_lv
HOST DEVICE int get_size() const
bool hasTimestampTypeFrameBound() const
llvm::Value * current_col_value_lv
bool isFrameNavigateWindowFunction() const
void codegenWindowFunctionStateInit(CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *aggregate_state)
int64_t * getNullValueEndPos() const
bool window_function_conditional_aggregate(const SqlWindowFunctionKind kind)
llvm::Value * current_partition_start_offset_lv
llvm::Value * current_row_pos_lv
llvm::Value * target_partition_rowid_ptr_lv
llvm::Value * codegenFrameBound(bool for_start_bound, bool for_range_mode, bool for_window_frame_naviation, const Analyzer::WindowFrame *frame_bound, bool is_timestamp_type_frame, llvm::Value *order_key_null_val, const WindowFrameBoundFuncArgs &args)
llvm::Value * frame_end_bound_expr_lv
llvm::Value * num_elem_current_partition_lv
llvm::Value * nulls_first_lv
const std::vector< SQLTypeInfo > & getOrderKeyColumnBufferTypes() const
llvm::Value * posArg(const Analyzer::Expr *) const
llvm::Value * aggregateWindowStatePtr(CodeGenerator *code_generator, const CompilationOptions &co)
size_t elementCount() const
llvm::Type * get_fp_type(const int width, llvm::LLVMContext &context)
std::string getFramingFuncName(const std::string &bound_type, const std::string &order_col_type, const std::string &op_type, bool for_timestamp_type) const
const int32_t * counts() const
const Analyzer::WindowFrame * getFrameStartBound() const
HOST DEVICE SQLTypes get_type() const
llvm::Value * target_partition_sorted_rowid_ptr_lv
const Analyzer::WindowFrame * getFrameEndBound() const
llvm::Value * target_partition_rowid_ptr_lv
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getOrderKeys() const
static WindowFunctionContext * getActiveWindowFunctionContext(Executor *executor)
std::string get_window_agg_name(const SqlWindowFunctionKind kind, const SQLTypeInfo &window_func_ti)
int64_t * getNullValueStartPos() const
const int64_t * partitionStartOffset() const
std::pair< llvm::Value *, llvm::Value * > codegenFrameNullRange(WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *partition_index_lv) const
size_t partitionCount() const
SqlWindowFrameBoundType getBoundType() const
llvm::Value * codegenWindowPosition(const WindowFunctionContext *window_func_context, llvm::Value *pos_arg)
llvm::Value * null_start_pos_lv
static const WindowProjectNodeContext * get(Executor *executor)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getArgs() const
int get_logical_size() const
std::vector< llvm::Value * > prepareRangeModeFuncArgs(bool for_start_bound, const Analyzer::WindowFrame *frame_bound, bool is_timestamp_type_frame, llvm::Value *order_key_null_val, const WindowFrameBoundFuncArgs &frame_args) const
const SQLTypeInfo getFirstOrderColTypeInfo(WindowFunctionContext *window_func_context) const
const WindowFunctionContext * activateWindowFunctionContext(Executor *executor, const size_t target_index) const
std::pair< llvm::Value *, llvm::Value * > codegenWindowFrameBounds(WindowFunctionContext *window_func_context, const Analyzer::WindowFrame *frame_start_bound, const Analyzer::WindowFrame *frame_end_bound, llvm::Value *order_key_col_null_val_lv, WindowFrameBoundFuncArgs &args, CodeGenerator &code_generator)
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
const Analyzer::Expr * getBoundExpr() const
llvm::ConstantFP * llFp(const float v) const
std::pair< llvm::BasicBlock *, llvm::Value * > codegenWindowResetStateControlFlow(CodeGenerator *code_generator, const CompilationOptions &co)
const SQLTypeInfo & get_type_info() const
static const int NUM_EXECUTION_DEVICES
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
llvm::Value * current_partition_start_offset_lv
llvm::Value * codegenLoadCurrentValueFromColBuf(WindowFunctionContext *window_func_context, CodeGenerator &code_generator, WindowFrameBoundFuncArgs &args) const
HOST DEVICE EncodingType get_compression() const
llvm::Value * target_partition_sorted_rowid_ptr_lv
void codegenWindowAvgEpilogue(CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *crt_val, llvm::Value *window_func_null_val)
const std::string getOrderKeyTypeName(WindowFunctionContext *window_func_context) const
llvm::Value * codegenConditionalAggregateCondValSelector(llvm::Value *cond_lv, SQLAgg const aggKind, CompilationOptions const &co) const
constexpr float inline_fp_null_value< float >()
std::pair< llvm::Value *, llvm::Value * > codegenFrameBoundRange(const Analyzer::WindowFunction *window_func, CodeGenerator &code_generator, const CompilationOptions &co)
const std::vector< const int8_t * > & getOrderKeyColumnBuffers() const
constexpr double inline_fp_null_value< double >()
llvm::Value * null_end_pos_lv
const int64_t * partitionNumCountBuf() const
llvm::Value * codegenWindowFunctionAggregate(CodeGenerator *code_generator, const CompilationOptions &co)
llvm::Value * toBool(llvm::Value *)
llvm::Value * codegenCurrentPartitionIndex(const WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *current_row_pos_lv)
HOST DEVICE int get_comp_param() const
llvm::Value * codegenWindowNavigationFunctionOnFrame(const CompilationOptions &co)
llvm::Value * codegenFrameBoundExpr(const Analyzer::WindowFunction *window_func, const Analyzer::WindowFrame *frame_bound, CodeGenerator &code_generator, const CompilationOptions &co)
llvm::ConstantInt * llInt(const T v) const
llvm::Value * order_key_buf_ptr_lv
llvm::Value * int64_t_zero_val_lv
const int64_t * sortedPartition() const
WindowPartitionBufferPtrs codegenLoadPartitionBuffers(WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *partition_index_lv) const
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::pair< std::string, llvm::Value * > codegenLoadOrderKeyBufPtr(WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co) const
const Analyzer::WindowFunction * getWindowFunction() const
bool is_dict_encoded_string() const
const int32_t * payload() const
llvm::Value * get_null_value_by_size(CgenState *cgen_state, SQLTypeInfo col_ti)
std::vector< llvm::Value * > createPtrWithHoistedMemoryAddr(CgenState *cgen_state, CodeGenerator *code_generator, CompilationOptions const &co, llvm::ConstantInt *ptr_int_val, llvm::Type *type, size_t num_devices_to_hoist_literal)
size_t getOrderKeySize(WindowFunctionContext *window_func_context) const
llvm::Value * codegenAggregateWindowState(CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *aggregate_state)
llvm::Value * int64_t_one_val_lv
llvm::Value * codegenWindowFunctionAggregateCalls(llvm::Value *aggregate_state, const CompilationOptions &co)
llvm::Value * frame_start_bound_expr_lv
llvm::Value * codegenWindowFunction(const size_t target_index, const CompilationOptions &co)
std::string order_type_col_name
llvm::Value * get_null_value_by_size_with_encoding(CgenState *cgen_state, SQLTypeInfo col_ti)
std::vector< llvm::Value * > prepareRowModeFuncArgs(bool for_start_bound, SqlWindowFrameBoundType bound_type, const WindowFrameBoundFuncArgs &args) const
SQLTypeInfo get_adjusted_window_type_info(const Analyzer::WindowFunction *window_func)