26 const auto window_func_context =
30 switch (window_func->getKind()) {
36 return code_generator.codegenWindowPosition(window_func_context,
37 code_generator.posArg(
nullptr));
42 return cgen_state_->emitCall(
"percent_window_func",
43 {cgen_state_->llInt(reinterpret_cast<const int64_t>(
44 window_func_context->output())),
45 code_generator.posArg(
nullptr)});
54 const auto&
args = window_func->getArgs();
56 const auto arg_lvs = code_generator.codegen(
args.front().get(),
true, co);
58 return arg_lvs.front();
68 return codegenWindowFunctionAggregate(co);
73 return codegenWindowNavigationFunctionOnFrame(co);
76 LOG(
FATAL) <<
"Invalid window function kind";
102 agg_name =
"agg_count";
106 agg_name =
"agg_count_if";
110 agg_name =
"agg_sum_if";
114 LOG(
FATAL) <<
"Invalid window function kind";
117 switch (window_func_ti.
get_type()) {
119 agg_name +=
"_float";
123 agg_name +=
"_double";
137 ?
args.front()->get_type_info()
150 }
else if (size == 2) {
152 }
else if (size == 4) {
159 if (col_ti.
is_fp()) {
167 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
170 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
176 return cgen_state->
llInt((int8_t)inline_int_null_value<int8_t>());
178 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
180 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
192 return cgen_state->
llInt((int64_t)inline_int_null_value<int64_t>());
202 if (col_ti.
is_fp()) {
209 llvm::Value* ret_val{
nullptr};
217 if (ret_val->getType()->getIntegerBitWidth() > ret_val_col_size_in_bytes) {
218 return cgen_state->
castToTypeIn(ret_val, ret_val_col_size_in_bytes);
228 const auto window_func_context =
230 const auto window_func = window_func_context->getWindowFunction();
233 arg_ti.get_type() ==
kFLOAT
234 ? llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0)
235 : llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
236 const auto aggregate_state_i64 = cgen_state_->llInt(
237 reinterpret_cast<const int64_t>(window_func_context->aggregateState()));
238 return cgen_state_->ir_builder_.CreateIntToPtr(aggregate_state_i64,
239 aggregate_state_type);
244 const auto reset_state_false_bb = codegenWindowResetStateControlFlow();
245 auto aggregate_state = aggregateWindowStatePtr();
246 llvm::Value* aggregate_state_count =
nullptr;
247 const auto window_func_context =
249 const auto window_func = window_func_context->getWindowFunction();
251 const auto aggregate_state_count_i64 = cgen_state_->llInt(
252 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
253 const auto pi64_type =
254 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
255 aggregate_state_count =
256 cgen_state_->ir_builder_.CreateIntToPtr(aggregate_state_count_i64, pi64_type);
258 codegenWindowFunctionStateInit(aggregate_state);
260 const auto count_zero = cgen_state_->llInt(int64_t(0));
261 cgen_state_->emitCall(
"agg_id", {aggregate_state_count, count_zero});
263 cgen_state_->ir_builder_.CreateBr(reset_state_false_bb);
264 cgen_state_->ir_builder_.SetInsertPoint(reset_state_false_bb);
266 return codegenWindowFunctionAggregateCalls(aggregate_state, co);
271 const auto window_func_context =
273 const auto bitset = cgen_state_->llInt(
274 reinterpret_cast<const int64_t>(window_func_context->partitionStart()));
275 const auto min_val = cgen_state_->llInt(int64_t(0));
276 const auto max_val = cgen_state_->llInt(window_func_context->elementCount() - 1);
277 const auto null_val = cgen_state_->llInt(inline_int_null_value<int64_t>());
278 const auto null_bool_val = cgen_state_->llInt<int8_t>(inline_int_null_value<int8_t>());
280 const auto reset_state =
281 code_generator.
toBool(cgen_state_->emitCall(
"bit_is_set",
283 code_generator.
posArg(
nullptr),
288 const auto reset_state_true_bb = llvm::BasicBlock::Create(
289 cgen_state_->context_,
"reset_state.true", cgen_state_->current_func_);
290 const auto reset_state_false_bb = llvm::BasicBlock::Create(
291 cgen_state_->context_,
"reset_state.false", cgen_state_->current_func_);
292 cgen_state_->ir_builder_.CreateCondBr(
293 reset_state, reset_state_true_bb, reset_state_false_bb);
294 cgen_state_->ir_builder_.SetInsertPoint(reset_state_true_bb);
295 return reset_state_false_bb;
300 const auto window_func_context =
302 const auto window_func = window_func_context->getWindowFunction();
304 const auto window_func_null_val =
305 window_func_ti.is_fp()
306 ? cgen_state_->inlineFpNull(window_func_ti)
307 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
308 llvm::Value* window_func_init_val;
309 const auto window_func_kind = window_func_context->getWindowFunction()->getKind();
312 switch (window_func_ti.get_type()) {
314 window_func_init_val = cgen_state_->llFp(
float(0));
318 window_func_init_val = cgen_state_->llFp(
double(0));
322 window_func_init_val = cgen_state_->llInt(int64_t(0));
327 window_func_init_val = window_func_null_val;
329 const auto pi32_type =
330 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
331 switch (window_func_ti.get_type()) {
333 cgen_state_->emitCall(
"agg_id_double", {aggregate_state, window_func_init_val});
338 cgen_state_->ir_builder_.CreateBitCast(aggregate_state, pi32_type);
339 cgen_state_->emitCall(
"agg_id_float", {aggregate_state, window_func_init_val});
343 cgen_state_->emitCall(
"agg_id", {aggregate_state, window_func_init_val});
352 const auto window_func_context =
354 const auto window_func = window_func_context->getWindowFunction();
355 const auto window_func_kind = window_func->getKind();
356 const auto&
args = window_func->getArgs();
360 const auto target_col_ti =
args.front()->get_type_info();
361 const auto target_col_size = target_col_ti.get_size();
362 const auto target_col_type_name =
365 window_func->get_type_info().get_size(), window_func->get_type_info().is_fp());
370 auto logical_null_val_lv =
372 auto target_col_null_val_lv =
374 if (window_func_context->elementCount() == 0) {
376 return target_col_null_val_lv;
379 auto [frame_start_bound_expr_lv, frame_end_bound_expr_lv] =
380 codegenFrameBoundRange(window_func, code_generator, co);
382 auto current_row_pos_lv = code_generator.posArg(
nullptr);
383 auto partition_index_lv =
384 codegenCurrentPartitionIndex(window_func_context, current_row_pos_lv);
387 size_t target_col_size_in_byte = target_col_size * 8;
389 target_col_ti.is_fp()
390 ?
get_fp_type(target_col_size_in_byte, cgen_state_->context_)
391 :
get_int_type(target_col_size_in_byte, cgen_state_->context_);
392 auto col_buf_type = llvm::PointerType::get(col_buf_ptr_type, 0);
393 auto target_col_buf_ptr_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
394 window_func_context->getColumnBufferForWindowFunctionExpressions().front()));
395 auto target_col_buf_lv =
396 cgen_state_->ir_builder_.CreateIntToPtr(target_col_buf_ptr_lv, col_buf_type);
399 auto partition_buf_ptrs =
400 codegenLoadPartitionBuffers(window_func_context, partition_index_lv);
403 const auto order_key_buf_ti =
404 window_func_context->getOrderKeyColumnBufferTypes().front();
405 auto const ordering_spec = window_func->getCollation().front();
406 auto order_key_col_null_val_lv =
410 auto [order_col_type_name, order_key_buf_ptr_lv] =
411 codegenLoadOrderKeyBufPtr(window_func_context);
414 auto [null_start_pos_lv, null_end_pos_lv] =
415 codegenFrameNullRange(window_func_context, partition_index_lv);
418 std::string row_idx_on_frame_func =
"compute_";
419 row_idx_on_frame_func += order_col_type_name;
420 row_idx_on_frame_func += ordering_spec.is_desc ?
"_greater_equal" :
"_less_equal";
421 row_idx_on_frame_func +=
"_current_row_idx_in_frame";
422 auto int64_t_one_val_lv = cgen_state_->llInt((int64_t)1);
423 auto nulls_first_lv = cgen_state_->llBool(ordering_spec.nulls_first);
424 auto cur_row_idx_in_frame_lv =
425 cgen_state_->emitCall(row_idx_on_frame_func,
426 {partition_buf_ptrs.num_elem_current_partition_lv,
428 order_key_buf_ptr_lv,
429 partition_buf_ptrs.target_partition_rowid_ptr_lv,
430 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
431 order_key_col_null_val_lv,
437 auto const int64_t_zero_val_lv = cgen_state_->llInt((int64_t)0);
439 frame_start_bound_expr_lv,
440 frame_end_bound_expr_lv,
441 window_func->hasRangeModeFraming() ? current_row_pos_lv : cur_row_idx_in_frame_lv,
443 window_func->hasRangeModeFraming()
444 ? int64_t_zero_val_lv
448 partition_buf_ptrs.num_elem_current_partition_lv,
449 order_key_buf_ptr_lv,
451 partition_buf_ptrs.target_partition_rowid_ptr_lv,
452 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
456 auto [frame_start_bound_lv, frame_end_bound_lv] =
457 codegenWindowFrameBounds(window_func_context,
458 window_func->getFrameStartBound(),
459 window_func->getFrameEndBound(),
460 order_key_col_null_val_lv,
465 llvm::Value* modified_cur_row_idx_in_frame_lv{
nullptr};
466 auto const offset_lv =
467 cgen_state_->castToTypeIn(code_generator.codegen(
args[1].get(),
true, co)[0], 64);
468 switch (window_func_kind) {
470 modified_cur_row_idx_in_frame_lv =
471 cgen_state_->ir_builder_.CreateSub(cur_row_idx_in_frame_lv, offset_lv);
474 modified_cur_row_idx_in_frame_lv =
475 cgen_state_->ir_builder_.CreateAdd(cur_row_idx_in_frame_lv, offset_lv);
478 auto candidate_row_idx =
479 cgen_state_->ir_builder_.CreateAdd(frame_start_bound_lv, offset_lv);
480 auto out_of_frame_bound_lv =
481 cgen_state_->ir_builder_.CreateICmpSGT(candidate_row_idx, frame_end_bound_lv);
483 modified_cur_row_idx_in_frame_lv = cgen_state_->ir_builder_.CreateSelect(
484 out_of_frame_bound_lv, cgen_state_->llInt((int64_t)-1), candidate_row_idx);
488 UNREACHABLE() <<
"Unsupported window function to navigate a window frame.";
490 CHECK(modified_cur_row_idx_in_frame_lv);
493 std::string target_func_name =
"get_";
494 target_func_name += target_col_type_name +
"_value_";
495 target_func_name += target_col_logical_type_name +
"_type_";
496 target_func_name +=
"in_frame";
498 cgen_state_->emitCall(target_func_name,
499 {modified_cur_row_idx_in_frame_lv,
500 frame_start_bound_lv,
503 partition_buf_ptrs.target_partition_rowid_ptr_lv,
504 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
506 target_col_null_val_lv});
508 res_lv = cgen_state_->emitCall(
510 {res_lv, logical_null_val_lv, cgen_state_->llInt((int64_t)
kSecsPerDay)});
524 const auto order_col_ti = window_func->
getOrderKeys().front()->get_type_info();
525 auto encode_date_col_val = [&order_col_ti,
this](llvm::Value* bound_expr_lv) {
526 if (order_col_ti.get_comp_param() == 16) {
527 return cgen_state_->emitCall(
528 "fixed_width_date_encode_noinline",
534 return cgen_state_->emitCall(
"fixed_width_date_encode_noinline",
540 llvm::Value* bound_expr_lv{
nullptr};
541 if (needs_bound_expr_codegen(frame_bound)) {
543 if (
auto dateadd_expr = dynamic_cast<const Analyzer::DateaddExpr*>(bound_expr)) {
544 if (dateadd_expr->get_datetime_expr()->get_type_info().is_encoded_timestamp()) {
545 dateadd_expr->set_fixed_encoding_null_val();
548 auto bound_expr_lvs = code_generator.
codegen(bound_expr,
true, co);
549 bound_expr_lv = bound_expr_lvs.front();
552 throw std::runtime_error(
553 "Range mode with date type ordering column is not supported yet.");
555 bound_expr_lv = encode_date_col_val(bound_expr_lv);
558 bound_expr_lv = cgen_state_->castToTypeIn(bound_expr_lv, 64);
561 bound_expr_lv = cgen_state_->llInt((int64_t)-1);
563 CHECK(bound_expr_lv);
564 return bound_expr_lv;
569 bool for_window_frame_naviation,
571 bool is_timestamp_type_frame,
572 llvm::Value* order_key_null_val,
575 auto adjust_frame_end_bound = [&](llvm::Value* target_bound_lv) {
579 CHECK(for_start_bound) <<
"frame end cannot be UNBOUNDED PRECEDING";
582 CHECK(!for_start_bound) <<
"frame start cannot be UNBOUNDED FOLLOWING";
584 return for_window_frame_naviation
588 std::vector<llvm::Value*> func_args;
589 std::string op_name =
591 if (!for_range_mode) {
592 llvm::Value* current_row_bound_expr_lv{
nullptr};
593 if (for_window_frame_naviation) {
598 current_row_bound_expr_lv =
601 current_row_bound_expr_lv =
608 if (for_start_bound) {
609 return cgen_state_->ir_builder_.CreateSelect(
610 cgen_state_->ir_builder_.CreateICmpSLT(current_row_bound_expr_lv,
613 current_row_bound_expr_lv);
615 return cgen_state_->ir_builder_.CreateSelect(
616 cgen_state_->ir_builder_.CreateICmpSGE(current_row_bound_expr_lv,
619 current_row_bound_expr_lv);
622 std::string func_class = for_start_bound ?
"start" :
"end";
623 auto const func_name =
"compute_row_mode_" + func_class +
"_index_" + op_name;
624 func_args = prepareRowModeFuncArgs(for_start_bound, bound_type, args);
625 current_row_bound_expr_lv = cgen_state_->emitCall(func_name, func_args);
627 return current_row_bound_expr_lv;
629 std::string func_class = for_start_bound ?
"lower" :
"upper";
630 auto const func_name = getFramingFuncName(
635 func_args = prepareRangeModeFuncArgs(
636 for_start_bound, frame_bound, is_timestamp_type_frame, order_key_null_val, args);
637 auto frame_bound_lv = cgen_state_->emitCall(func_name, func_args);
638 if (!for_start_bound && for_window_frame_naviation) {
640 frame_bound_lv = cgen_state_->ir_builder_.CreateSelect(
641 cgen_state_->ir_builder_.CreateICmpSGE(frame_bound_lv,
646 return frame_bound_lv;
653 return window_func->
getOrderKeys().front()->get_type_info();
657 const auto order_key_size = getFirstOrderColTypeInfo(window_func_context).get_size();
658 return order_key_size;
663 auto const order_key_size = getOrderKeySize(window_func_context);
664 auto const order_key_ptr =
666 CHECK(order_key_ptr);
668 order_key_ptr->get_type_info().is_fp());
675 llvm::Value* current_col_value_ptr_lv{
nullptr};
676 const auto order_key_size_in_byte = getOrderKeySize(window_func_context) * 8;
677 auto const order_key_ptr =
679 CHECK(order_key_ptr);
680 auto const order_col_ti = order_key_ptr->get_type_info();
681 auto const order_col_llvm_type =
682 order_col_ti.is_fp() ?
get_fp_type(order_key_size_in_byte, cgen_state_->context_)
683 :
get_int_type(order_key_size_in_byte, cgen_state_->context_);
687 current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
690 current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
693 return cgen_state_->ir_builder_.CreateLoad(
694 current_col_value_ptr_lv->getType()->getPointerElementType(),
695 current_col_value_ptr_lv,
696 "current_col_value");
701 llvm::Value* current_row_pos_lv) {
702 const auto pi64_type =
703 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
704 const auto pi32_type =
705 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
706 auto row_pos_lv = current_row_pos_lv;
724 auto*
const hash_slot_idx_ptr =
726 auto hash_slot_idx_buf_lv =
727 cgen_state_->llInt(reinterpret_cast<int64_t>(hash_slot_idx_ptr));
728 auto hash_slot_idx_ptr_lv =
729 cgen_state_->ir_builder_.CreateIntToPtr(hash_slot_idx_buf_lv, pi32_type);
730 auto hash_slot_idx_load_lv = cgen_state_->ir_builder_.CreateGEP(
731 hash_slot_idx_ptr_lv->getType()->getPointerElementType(),
732 hash_slot_idx_ptr_lv,
734 row_pos_lv = cgen_state_->castToTypeIn(
735 cgen_state_->ir_builder_.CreateLoad(
736 hash_slot_idx_load_lv->getType()->getPointerElementType(),
737 hash_slot_idx_load_lv,
738 "cur_row_hash_slot_idx"),
741 auto partition_count_lv = cgen_state_->llInt(window_func_context->
partitionCount());
742 auto partition_num_count_buf_lv = cgen_state_->llInt(
744 auto partition_num_count_ptr_lv =
745 cgen_state_->ir_builder_.CreateIntToPtr(partition_num_count_buf_lv, pi64_type);
746 return cgen_state_->emitCall(
747 "compute_int64_t_lower_bound",
748 {partition_count_lv, row_pos_lv, partition_num_count_ptr_lv});
752 const std::string& order_col_type,
753 const std::string& op_type,
754 bool for_timestamp_type)
const {
755 auto target_val_type = for_timestamp_type ?
"int64_t" : order_col_type;
756 auto null_type = for_timestamp_type ?
"int64_t" : order_col_type;
757 return "range_mode_" + target_val_type +
"_" + order_col_type +
"_" + null_type +
"_" +
758 op_type +
"_frame_" + bound_type +
"_bound";
762 bool for_start_bound,
780 bool for_start_bound,
782 bool is_timestamp_type_frame,
783 llvm::Value* order_key_null_val,
785 llvm::Value* bound_expr_lv =
787 llvm::Value* target_val_lv =
791 llvm::Value* frame_bound_val_lv =
810 llvm::Value* partition_index_lv)
const {
811 const auto pi64_type =
812 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
813 const auto null_start_pos_buf = cgen_state_->llInt(
815 const auto null_start_pos_buf_ptr =
816 cgen_state_->ir_builder_.CreateIntToPtr(null_start_pos_buf, pi64_type);
817 const auto null_start_pos_ptr =
818 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
819 null_start_pos_buf_ptr,
821 auto null_start_pos_lv = cgen_state_->ir_builder_.CreateLoad(
822 null_start_pos_ptr->getType()->getPointerElementType(),
825 const auto null_end_pos_buf = cgen_state_->llInt(
827 const auto null_end_pos_buf_ptr =
828 cgen_state_->ir_builder_.CreateIntToPtr(null_end_pos_buf, pi64_type);
829 const auto null_end_pos_ptr = cgen_state_->ir_builder_.CreateGEP(
830 get_int_type(64, cgen_state_->context_), null_end_pos_buf_ptr, partition_index_lv);
831 auto null_end_pos_lv = cgen_state_->ir_builder_.CreateLoad(
832 null_end_pos_ptr->getType()->getPointerElementType(),
835 return std::make_pair(null_start_pos_lv, null_end_pos_lv);
840 auto const order_key_ti =
842 auto const order_key_size = order_key_ti.get_size();
846 size_t order_key_size_in_byte = order_key_size * 8;
847 auto const order_key_type =
848 order_key_ti.is_fp() ?
get_fp_type(order_key_size_in_byte, cgen_state_->context_)
849 :
get_int_type(order_key_size_in_byte, cgen_state_->context_);
850 auto const order_key_buf_type = llvm::PointerType::get(order_key_type, 0);
851 auto const order_key_buf = cgen_state_->llInt(
853 auto const order_key_buf_ptr_lv =
854 cgen_state_->ir_builder_.CreateIntToPtr(order_key_buf, order_key_buf_type);
855 return std::make_pair(order_col_type_name, order_key_buf_ptr_lv);
860 llvm::Value* partition_index_lv)
const {
862 const auto pi64_type =
863 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
864 const auto pi32_type =
865 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
868 auto partition_start_offset_buf_lv = cgen_state_->llInt(
870 auto partition_start_offset_ptr_lv =
871 cgen_state_->ir_builder_.CreateIntToPtr(partition_start_offset_buf_lv, pi64_type);
874 auto current_partition_start_offset_ptr_lv =
875 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
876 partition_start_offset_ptr_lv,
879 current_partition_start_offset_ptr_lv->getType()->getPointerElementType(),
880 current_partition_start_offset_ptr_lv);
883 const auto partition_rowid_buf_lv =
884 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->
payload()));
885 const auto partition_rowid_ptr_lv =
886 cgen_state_->ir_builder_.CreateIntToPtr(partition_rowid_buf_lv, pi32_type);
888 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
889 partition_rowid_ptr_lv,
893 const auto sorted_rowid_lv = cgen_state_->llInt(
895 const auto sorted_rowid_ptr_lv =
896 cgen_state_->ir_builder_.CreateIntToPtr(sorted_rowid_lv, pi64_type);
898 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
903 const auto partition_count_buf =
904 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->
counts()));
905 auto partition_count_buf_ptr_lv =
906 cgen_state_->ir_builder_.CreateIntToPtr(partition_count_buf, pi32_type);
909 const auto num_elem_current_partition_ptr =
910 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
911 partition_count_buf_ptr_lv,
914 cgen_state_->ir_builder_.CreateLoad(
915 num_elem_current_partition_ptr->getType()->getPointerElementType(),
916 num_elem_current_partition_ptr),
927 auto frame_start_bound_expr_lv =
928 codegenFrameBoundExpr(window_func, frame_start_bound, code_generator, co);
929 auto frame_end_bound_expr_lv =
930 codegenFrameBoundExpr(window_func, frame_end_bound, code_generator, co);
931 CHECK(frame_start_bound_expr_lv);
932 CHECK(frame_end_bound_expr_lv);
933 return std::make_pair(frame_start_bound_expr_lv, frame_end_bound_expr_lv);
940 llvm::Value* order_key_col_null_val_lv,
948 if (window_func->hasRangeModeFraming()) {
950 CHECK(window_func->getOrderKeys().size() == 1UL);
954 codegenLoadCurrentValueFromColBuf(window_func_context, code_generator, args);
957 auto get_order_key_null_val = [is_timestamp_type_frame,
958 &order_key_col_null_val_lv,
960 return is_timestamp_type_frame && !frame_bound->isCurrentRowBound()
961 ? cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64)
962 : order_key_col_null_val_lv;
964 auto frame_start_bound_lv =
965 codegenFrameBound(
true,
966 window_func->hasRangeModeFraming(),
967 window_func->isFrameNavigateWindowFunction(),
969 is_timestamp_type_frame,
970 get_order_key_null_val(frame_start_bound),
972 auto frame_end_bound_lv =
973 codegenFrameBound(
false,
974 window_func->hasRangeModeFraming(),
975 window_func->isFrameNavigateWindowFunction(),
977 is_timestamp_type_frame,
978 get_order_key_null_val(frame_end_bound),
980 CHECK(frame_start_bound_lv);
981 CHECK(frame_end_bound_lv);
982 return std::make_pair(frame_start_bound_lv, frame_end_bound_lv);
988 const auto window_func_context =
990 const auto window_func = window_func_context->getWindowFunction();
992 const auto window_func_null_val =
993 window_func_ti.is_fp()
994 ? cgen_state_->inlineFpNull(window_func_ti)
995 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
996 if (window_func_context->elementCount() == 0) {
1000 : window_func_null_val;
1002 const auto&
args = window_func->getArgs();
1004 if (window_func_context->needsToBuildAggregateTree()) {
1008 const auto pi64_type =
1009 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1010 const auto ppi64_type = llvm::PointerType::get(
1011 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0), 0);
1013 auto [frame_start_bound_expr_lv, frame_end_bound_expr_lv] =
1014 codegenFrameBoundRange(window_func, code_generator, co);
1017 auto current_row_pos_lv = code_generator.
posArg(
nullptr);
1018 auto partition_index_lv =
1019 codegenCurrentPartitionIndex(window_func_context, current_row_pos_lv);
1022 const auto target_col_ti =
args.front()->get_type_info();
1023 const auto target_col_size = target_col_ti.get_size();
1024 const auto col_type_name =
1027 const auto partition_buf_ptrs =
1028 codegenLoadPartitionBuffers(window_func_context, partition_index_lv);
1030 auto [order_col_type_name, order_key_buf_ptr_lv] =
1031 codegenLoadOrderKeyBufPtr(window_func_context);
1034 const auto order_key_buf_ti =
1035 window_func_context->getOrderKeyColumnBufferTypes().front();
1036 auto const ordering_spec = window_func->getCollation().front();
1037 llvm::Value* order_key_col_null_val_lv{
nullptr};
1038 switch (order_key_buf_ti.get_type()) {
1045 order_key_col_null_val_lv = cgen_state_->llInt((int32_t)null_val);
1050 order_key_col_null_val_lv = cgen_state_->inlineNull(order_key_buf_ti);
1055 auto [null_start_pos_lv, null_end_pos_lv] =
1056 codegenFrameNullRange(window_func_context, partition_index_lv);
1057 auto nulls_first_lv = cgen_state_->llBool(ordering_spec.nulls_first);
1060 frame_start_bound_expr_lv,
1061 frame_end_bound_expr_lv,
1065 cgen_state_->llInt((int64_t)0),
1066 cgen_state_->llInt((int64_t)1),
1067 partition_buf_ptrs.num_elem_current_partition_lv,
1068 order_key_buf_ptr_lv,
1070 partition_buf_ptrs.target_partition_rowid_ptr_lv,
1071 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
1075 auto [frame_start_bound_lv, frame_end_bound_lv] =
1076 codegenWindowFrameBounds(window_func_context,
1077 window_func->getFrameStartBound(),
1078 window_func->getFrameEndBound(),
1079 order_key_col_null_val_lv,
1084 llvm::Value* aggregation_trees_lv{
nullptr};
1085 llvm::Value* invalid_val_lv{
nullptr};
1086 llvm::Value* null_val_lv{
nullptr};
1087 std::string aggregation_tree_search_func_name{
"search_"};
1088 std::string aggregation_tree_getter_func_name{
"get_"};
1092 auto agg_expr_ti =
args.front()->get_type_info();
1093 if (agg_expr_ti.is_fp()) {
1095 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::max());
1097 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::lowest());
1099 invalid_val_lv = cgen_state_->llFp((
double)0);
1102 aggregation_tree_search_func_name +=
"double";
1103 aggregation_tree_getter_func_name +=
"double";
1106 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::max());
1108 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::lowest());
1110 invalid_val_lv = cgen_state_->llInt((int64_t)0);
1112 null_val_lv = cgen_state_->llInt(inline_int_null_value<int64_t>());
1113 aggregation_tree_search_func_name +=
"int64_t";
1114 aggregation_tree_getter_func_name +=
"integer";
1119 aggregation_tree_search_func_name +=
"_derived";
1120 aggregation_tree_getter_func_name +=
"_derived";
1124 if (agg_expr_ti.is_fp()) {
1126 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1127 window_func_context->getDerivedAggregationTreesForDoubleTypeWindowExpr()));
1129 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1130 window_func_context->getAggregationTreesForDoubleTypeWindowExpr()));
1134 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1135 window_func_context->getDerivedAggregationTreesForIntegerTypeWindowExpr()));
1137 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1138 window_func_context->getAggregationTreesForIntegerTypeWindowExpr()));
1142 CHECK(aggregation_trees_lv);
1143 CHECK(invalid_val_lv);
1144 aggregation_tree_search_func_name +=
"_aggregation_tree";
1145 aggregation_tree_getter_func_name +=
"_aggregation_tree";
1148 auto aggregation_trees_ptr =
1149 cgen_state_->ir_builder_.CreateIntToPtr(aggregation_trees_lv, ppi64_type);
1150 auto target_aggregation_tree_lv = cgen_state_->emitCall(
1151 aggregation_tree_getter_func_name, {aggregation_trees_ptr, partition_index_lv});
1154 const auto tree_depth_buf = cgen_state_->llInt(
1155 reinterpret_cast<int64_t>(window_func_context->getAggregateTreeDepth()));
1156 const auto tree_depth_buf_ptr =
1157 cgen_state_->ir_builder_.CreateIntToPtr(tree_depth_buf, pi64_type);
1158 const auto current_partition_tree_depth_buf_ptr = cgen_state_->ir_builder_.CreateGEP(
1159 get_int_type(64, cgen_state_->context_), tree_depth_buf_ptr, partition_index_lv);
1160 const auto current_partition_tree_depth_lv = cgen_state_->ir_builder_.CreateLoad(
1161 current_partition_tree_depth_buf_ptr->getType()->getPointerElementType(),
1162 current_partition_tree_depth_buf_ptr);
1165 const auto aggregation_tree_fanout_lv = cgen_state_->llInt(
1166 static_cast<int64_t>(window_func_context->getAggregateTreeFanout()));
1169 const auto agg_type_lv =
1170 cgen_state_->llInt(static_cast<int32_t>(window_func->getKind()));
1175 cgen_state_->emitCall(aggregation_tree_search_func_name,
1176 {target_aggregation_tree_lv,
1177 frame_start_bound_lv,
1179 current_partition_tree_depth_lv,
1180 aggregation_tree_fanout_lv,
1181 cgen_state_->llBool(agg_expr_ti.is_decimal()),
1182 cgen_state_->llInt((int64_t)agg_expr_ti.get_scale()),
1188 std::string null_handler_func_name{
"handle_null_val_"};
1189 std::vector<llvm::Value*> null_handler_args{res_lv, null_val_lv};
1195 if (agg_expr_ti.is_fp()) {
1197 null_handler_func_name +=
"double_double";
1200 null_handler_func_name +=
"double_int64_t";
1202 }
else if (agg_expr_ti.is_fp()) {
1204 null_handler_func_name +=
"double_double";
1207 null_handler_func_name +=
"int64_t_int64_t";
1209 null_handler_func_name +=
"_window_framing_agg";
1213 if (agg_expr_ti.is_fp()) {
1214 null_handler_args.push_back(cgen_state_->llFp((
double)0));
1216 null_handler_args.push_back(cgen_state_->llInt((int64_t)0));
1221 null_handler_args.push_back(cgen_state_->castToTypeIn(window_func_null_val, 64));
1223 res_lv = cgen_state_->emitCall(null_handler_func_name, null_handler_args);
1228 return cgen_state_->ir_builder_.CreateFPToSI(
1235 std::vector<llvm::Value*> agg_func_args{aggregate_state};
1236 auto modified_window_func_null_val = window_func_null_val;
1241 agg_func_args.push_back(cgen_state_->llInt(int64_t(1)));
1245 arg_target_expr =
args.front().get();
1246 const auto arg_lvs = code_generator.
codegen(arg_target_expr,
true, co);
1247 CHECK_EQ(arg_lvs.size(), size_t(1));
1249 auto crt_val = arg_lvs.front();
1252 !window_func_ti.is_fp()) {
1254 arg_lvs.front(),
args.front()->get_type_info(), window_func_ti,
false);
1256 agg_func_args.push_back(window_func_ti.get_type() ==
kFLOAT
1258 : cgen_state_->castToTypeIn(crt_val, 64));
1260 llvm::Value* cond_lv{
nullptr};
1262 switch (window_func->getKind()) {
1265 modified_window_func_null_val = cgen_state_->castToTypeIn(
1271 if (
args.front()->get_type_info().is_integer()) {
1272 agg_func_args[1] = cgen_state_->castToTypeIn(agg_func_args[1], 64);
1274 modified_window_func_null_val =
1275 cgen_state_->castToTypeIn(window_func_null_val, 64);
1277 auto cond_expr_lv = code_generator.
codegen(
args[1].
get(),
true, co).front();
1279 codegenConditionalAggregateCondValSelector(cond_expr_lv,
kSUM_IF, co);
1285 agg_name +=
"_skip_val";
1286 agg_func_args.push_back(modified_window_func_null_val);
1288 agg_func_args.push_back(cond_lv);
1291 cgen_state_->emitCall(agg_name, agg_func_args);
1293 codegenWindowAvgEpilogue(agg_func_args[1], window_func_null_val);
1295 return codegenAggregateWindowState();
1300 llvm::Value* window_func_null_val) {
1302 const auto window_func_context =
1304 const auto window_func = window_func_context->getWindowFunction();
1306 const auto pi32_type =
1307 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1308 const auto pi64_type =
1309 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1310 const auto aggregate_state_type =
1311 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1312 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1313 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1314 auto aggregate_state_count = cgen_state_->ir_builder_.CreateIntToPtr(
1315 aggregate_state_count_i64, aggregate_state_type);
1316 std::string agg_count_func_name =
"agg_count";
1317 switch (window_func_ti.get_type()) {
1319 agg_count_func_name +=
"_float";
1323 agg_count_func_name +=
"_double";
1330 agg_count_func_name +=
"_skip_val";
1331 cgen_state_->emitCall(agg_count_func_name,
1332 {aggregate_state_count, crt_val, window_func_null_val});
1337 const auto pi32_type =
1338 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1339 const auto pi64_type =
1340 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1341 const auto window_func_context =
1345 const auto aggregate_state_type =
1346 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1347 auto aggregate_state = aggregateWindowStatePtr();
1349 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1350 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1351 auto aggregate_state_count = cgen_state_->ir_builder_.CreateIntToPtr(
1352 aggregate_state_count_i64, aggregate_state_type);
1354 switch (window_func_ti.get_type()) {
1356 return cgen_state_->emitCall(
1357 "load_avg_float", {aggregate_state, aggregate_state_count, double_null_lv});
1360 return cgen_state_->emitCall(
1361 "load_avg_double", {aggregate_state, aggregate_state_count, double_null_lv});
1364 return cgen_state_->emitCall(
1367 aggregate_state_count,
1369 cgen_state_->llInt<int32_t>(window_func_ti.get_scale())});
1372 return cgen_state_->emitCall(
1373 "load_avg_int", {aggregate_state, aggregate_state_count, double_null_lv});
1378 return cgen_state_->ir_builder_.CreateLoad(
1379 aggregate_state->getType()->getPointerElementType(), aggregate_state);
1381 switch (window_func_ti.get_type()) {
1383 return cgen_state_->emitCall(
"load_float", {aggregate_state});
1386 return cgen_state_->emitCall(
"load_double", {aggregate_state});
1389 return cgen_state_->ir_builder_.CreateLoad(
1390 aggregate_state->getType()->getPointerElementType(), aggregate_state);
1396 llvm::Value* cond_lv,
1399 llvm::Value* res_cond_lv{
nullptr};
1402 if (cond_lv->getType()->isIntegerTy(1)) {
1405 res_cond_lv = cgen_state_->castToTypeIn(cond_lv, 8);
1407 CHECK(cond_lv->getType()->isIntegerTy(8));
1413 cgen_state_->ir_builder_.CreateICmpEQ(cond_lv, cgen_state_->llInt((int8_t)1));
1414 res_cond_lv = cgen_state_->ir_builder_.CreateSelect(
1415 true_cond_lv, cgen_state_->llInt((int8_t)1), cgen_state_->llInt((int8_t)0));
std::string get_col_type_name_by_size(const size_t size, const bool is_fp)
bool hasRangeModeFraming() const
bool isCurrentRowBound() const
llvm::Value * castToTypeIn(llvm::Value *val, const size_t bit_width)
SqlWindowFunctionKind getKind() const
llvm::Value * num_elem_current_partition_lv
HOST DEVICE int get_size() const
bool hasTimestampTypeFrameBound() const
std::pair< std::string, llvm::Value * > codegenLoadOrderKeyBufPtr(WindowFunctionContext *window_func_context) const
std::pair< llvm::Value *, llvm::Value * > codegenFrameNullRange(WindowFunctionContext *window_func_context, llvm::Value *partition_index_lv) const
llvm::Value * current_col_value_lv
bool isFrameNavigateWindowFunction() const
llvm::Value * codegenCurrentPartitionIndex(const WindowFunctionContext *window_func_context, llvm::Value *current_row_pos_lv)
int64_t * getNullValueEndPos() const
bool window_function_conditional_aggregate(const SqlWindowFunctionKind kind)
llvm::Value * current_partition_start_offset_lv
llvm::Value * current_row_pos_lv
llvm::Value * target_partition_rowid_ptr_lv
llvm::Value * codegenFrameBound(bool for_start_bound, bool for_range_mode, bool for_window_frame_naviation, const Analyzer::WindowFrame *frame_bound, bool is_timestamp_type_frame, llvm::Value *order_key_null_val, const WindowFrameBoundFuncArgs &args)
llvm::Value * frame_end_bound_expr_lv
llvm::Value * num_elem_current_partition_lv
llvm::Value * nulls_first_lv
const std::vector< SQLTypeInfo > & getOrderKeyColumnBufferTypes() const
llvm::Value * posArg(const Analyzer::Expr *) const
llvm::Value * aggregateWindowStatePtr()
size_t elementCount() const
llvm::Type * get_fp_type(const int width, llvm::LLVMContext &context)
std::string getFramingFuncName(const std::string &bound_type, const std::string &order_col_type, const std::string &op_type, bool for_timestamp_type) const
const int32_t * counts() const
const Analyzer::WindowFrame * getFrameStartBound() const
HOST DEVICE SQLTypes get_type() const
llvm::Value * target_partition_sorted_rowid_ptr_lv
const Analyzer::WindowFrame * getFrameEndBound() const
llvm::Value * target_partition_rowid_ptr_lv
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getOrderKeys() const
static WindowFunctionContext * getActiveWindowFunctionContext(Executor *executor)
std::string get_window_agg_name(const SqlWindowFunctionKind kind, const SQLTypeInfo &window_func_ti)
int64_t * getNullValueStartPos() const
const int64_t * partitionStartOffset() const
size_t partitionCount() const
SqlWindowFrameBoundType getBoundType() const
llvm::Value * codegenWindowPosition(const WindowFunctionContext *window_func_context, llvm::Value *pos_arg)
llvm::Value * null_start_pos_lv
static const WindowProjectNodeContext * get(Executor *executor)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getArgs() const
int get_logical_size() const
std::vector< llvm::Value * > prepareRangeModeFuncArgs(bool for_start_bound, const Analyzer::WindowFrame *frame_bound, bool is_timestamp_type_frame, llvm::Value *order_key_null_val, const WindowFrameBoundFuncArgs &frame_args) const
const SQLTypeInfo getFirstOrderColTypeInfo(WindowFunctionContext *window_func_context) const
const WindowFunctionContext * activateWindowFunctionContext(Executor *executor, const size_t target_index) const
std::pair< llvm::Value *, llvm::Value * > codegenWindowFrameBounds(WindowFunctionContext *window_func_context, const Analyzer::WindowFrame *frame_start_bound, const Analyzer::WindowFrame *frame_end_bound, llvm::Value *order_key_col_null_val_lv, WindowFrameBoundFuncArgs &args, CodeGenerator &code_generator)
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
const Analyzer::Expr * getBoundExpr() const
llvm::ConstantFP * llFp(const float v) const
const SQLTypeInfo & get_type_info() const
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
llvm::Value * current_partition_start_offset_lv
llvm::Value * codegenLoadCurrentValueFromColBuf(WindowFunctionContext *window_func_context, CodeGenerator &code_generator, WindowFrameBoundFuncArgs &args) const
HOST DEVICE EncodingType get_compression() const
llvm::Value * target_partition_sorted_rowid_ptr_lv
const std::string getOrderKeyTypeName(WindowFunctionContext *window_func_context) const
llvm::Value * codegenConditionalAggregateCondValSelector(llvm::Value *cond_lv, SQLAgg const aggKind, CompilationOptions const &co) const
void codegenWindowFunctionStateInit(llvm::Value *aggregate_state)
constexpr float inline_fp_null_value< float >()
std::pair< llvm::Value *, llvm::Value * > codegenFrameBoundRange(const Analyzer::WindowFunction *window_func, CodeGenerator &code_generator, const CompilationOptions &co)
const std::vector< const int8_t * > & getOrderKeyColumnBuffers() const
void codegenWindowAvgEpilogue(llvm::Value *crt_val, llvm::Value *window_func_null_val)
constexpr double inline_fp_null_value< double >()
llvm::Value * null_end_pos_lv
const int64_t * partitionNumCountBuf() const
llvm::Value * toBool(llvm::Value *)
llvm::Value * codegenAggregateWindowState()
llvm::Value * codegenWindowNavigationFunctionOnFrame(const CompilationOptions &co)
llvm::Value * codegenFrameBoundExpr(const Analyzer::WindowFunction *window_func, const Analyzer::WindowFrame *frame_bound, CodeGenerator &code_generator, const CompilationOptions &co)
llvm::ConstantInt * llInt(const T v) const
llvm::Value * codegenWindowFunctionAggregate(const CompilationOptions &co)
llvm::Value * order_key_buf_ptr_lv
llvm::Value * int64_t_zero_val_lv
const int64_t * sortedPartition() const
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
const Analyzer::WindowFunction * getWindowFunction() const
bool is_dict_encoded_string() const
const int32_t * payload() const
llvm::Value * get_null_value_by_size(CgenState *cgen_state, SQLTypeInfo col_ti)
size_t getOrderKeySize(WindowFunctionContext *window_func_context) const
llvm::Value * int64_t_one_val_lv
WindowPartitionBufferPtrs codegenLoadPartitionBuffers(WindowFunctionContext *window_func_context, llvm::Value *partition_index_lv) const
llvm::Value * codegenWindowFunctionAggregateCalls(llvm::Value *aggregate_state, const CompilationOptions &co)
llvm::Value * frame_start_bound_expr_lv
llvm::Value * codegenWindowFunction(const size_t target_index, const CompilationOptions &co)
std::string order_type_col_name
llvm::BasicBlock * codegenWindowResetStateControlFlow()
llvm::Value * get_null_value_by_size_with_encoding(CgenState *cgen_state, SQLTypeInfo col_ti)
std::vector< llvm::Value * > prepareRowModeFuncArgs(bool for_start_bound, SqlWindowFrameBoundType bound_type, const WindowFrameBoundFuncArgs &args) const
SQLTypeInfo get_adjusted_window_type_info(const Analyzer::WindowFunction *window_func)