26 const auto window_func_context =
30 switch (window_func->getKind()) {
36 return code_generator.codegenWindowPosition(window_func_context,
37 code_generator.posArg(
nullptr));
42 return cgen_state_->emitCall(
"percent_window_func",
43 {cgen_state_->llInt(reinterpret_cast<const int64_t>(
44 window_func_context->output())),
45 code_generator.posArg(
nullptr)});
53 const auto&
args = window_func->getArgs();
55 const auto arg_lvs = code_generator.codegen(
args.front().get(),
true, co);
57 return arg_lvs.front();
65 return codegenWindowFunctionAggregate(co);
69 return codegenWindowFunctionOnFrame(co);
72 LOG(
FATAL) <<
"Invalid window function kind";
98 agg_name =
"agg_count";
102 LOG(
FATAL) <<
"Invalid window function kind";
105 switch (window_func_ti.
get_type()) {
107 agg_name +=
"_float";
111 agg_name +=
"_double";
125 ?
args.front()->get_type_info()
138 }
else if (size == 2) {
140 }
else if (size == 4) {
147 if (col_ti.
is_fp()) {
155 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
158 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
164 return cgen_state->
llInt((int8_t)inline_int_null_value<int8_t>());
166 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
168 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
177 return cgen_state->
llInt((int64_t)inline_int_null_value<int64_t>());
187 if (col_ti.
is_fp()) {
194 llvm::Value* ret_val{
nullptr};
202 if (ret_val->getType()->getIntegerBitWidth() > ret_val_col_size_in_bytes) {
203 return cgen_state->
castToTypeIn(ret_val, ret_val_col_size_in_bytes);
213 const auto window_func_context =
215 const auto window_func = window_func_context->getWindowFunction();
218 arg_ti.get_type() ==
kFLOAT
219 ? llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0)
220 : llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
221 const auto aggregate_state_i64 = cgen_state_->llInt(
222 reinterpret_cast<const int64_t>(window_func_context->aggregateState()));
223 return cgen_state_->ir_builder_.CreateIntToPtr(aggregate_state_i64,
224 aggregate_state_type);
229 const auto reset_state_false_bb = codegenWindowResetStateControlFlow();
230 auto aggregate_state = aggregateWindowStatePtr();
231 llvm::Value* aggregate_state_count =
nullptr;
232 const auto window_func_context =
234 const auto window_func = window_func_context->getWindowFunction();
236 const auto aggregate_state_count_i64 = cgen_state_->llInt(
237 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
238 const auto pi64_type =
239 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
240 aggregate_state_count =
241 cgen_state_->ir_builder_.CreateIntToPtr(aggregate_state_count_i64, pi64_type);
243 codegenWindowFunctionStateInit(aggregate_state);
245 const auto count_zero = cgen_state_->llInt(int64_t(0));
246 cgen_state_->emitCall(
"agg_id", {aggregate_state_count, count_zero});
248 cgen_state_->ir_builder_.CreateBr(reset_state_false_bb);
249 cgen_state_->ir_builder_.SetInsertPoint(reset_state_false_bb);
251 return codegenWindowFunctionAggregateCalls(aggregate_state, co);
256 const auto window_func_context =
258 const auto bitset = cgen_state_->llInt(
259 reinterpret_cast<const int64_t>(window_func_context->partitionStart()));
260 const auto min_val = cgen_state_->llInt(int64_t(0));
261 const auto max_val = cgen_state_->llInt(window_func_context->elementCount() - 1);
262 const auto null_val = cgen_state_->llInt(inline_int_null_value<int64_t>());
263 const auto null_bool_val = cgen_state_->llInt<int8_t>(inline_int_null_value<int8_t>());
265 const auto reset_state =
266 code_generator.
toBool(cgen_state_->emitCall(
"bit_is_set",
268 code_generator.
posArg(
nullptr),
273 const auto reset_state_true_bb = llvm::BasicBlock::Create(
274 cgen_state_->context_,
"reset_state.true", cgen_state_->current_func_);
275 const auto reset_state_false_bb = llvm::BasicBlock::Create(
276 cgen_state_->context_,
"reset_state.false", cgen_state_->current_func_);
277 cgen_state_->ir_builder_.CreateCondBr(
278 reset_state, reset_state_true_bb, reset_state_false_bb);
279 cgen_state_->ir_builder_.SetInsertPoint(reset_state_true_bb);
280 return reset_state_false_bb;
285 const auto window_func_context =
287 const auto window_func = window_func_context->getWindowFunction();
289 const auto window_func_null_val =
290 window_func_ti.is_fp()
291 ? cgen_state_->inlineFpNull(window_func_ti)
292 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
293 llvm::Value* window_func_init_val;
294 if (window_func_context->getWindowFunction()->getKind() ==
296 switch (window_func_ti.get_type()) {
298 window_func_init_val = cgen_state_->llFp(
float(0));
302 window_func_init_val = cgen_state_->llFp(
double(0));
306 window_func_init_val = cgen_state_->llInt(int64_t(0));
311 window_func_init_val = window_func_null_val;
313 const auto pi32_type =
314 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
315 switch (window_func_ti.get_type()) {
317 cgen_state_->emitCall(
"agg_id_double", {aggregate_state, window_func_init_val});
322 cgen_state_->ir_builder_.CreateBitCast(aggregate_state, pi32_type);
323 cgen_state_->emitCall(
"agg_id_float", {aggregate_state, window_func_init_val});
327 cgen_state_->emitCall(
"agg_id", {aggregate_state, window_func_init_val});
335 const auto window_func_context =
337 const auto window_func = window_func_context->getWindowFunction();
338 const auto window_func_kind = window_func->getKind();
339 const auto&
args = window_func->getArgs();
342 const auto offset_lv =
343 cgen_state_->castToTypeIn(code_generator.codegen(
args[1].get(),
true, co)[0], 64);
346 const auto frame_start_bound = window_func->getFrameStartBound();
347 const auto frame_end_bound = window_func->getFrameEndBound();
348 auto frame_start_bound_expr_lv =
349 codegenFrameBoundExpr(window_func, frame_start_bound, code_generator, co);
350 auto frame_end_bound_expr_lv =
351 codegenFrameBoundExpr(window_func, frame_end_bound, code_generator, co);
352 CHECK(frame_start_bound_expr_lv);
353 CHECK(frame_end_bound_expr_lv);
355 auto current_row_pos_lv = code_generator.posArg(
nullptr);
356 auto partition_index_lv =
357 codegenCurrentPartitionIndex(window_func_context, current_row_pos_lv);
359 llvm::Value* res_lv{
nullptr};
364 const auto pi32_type =
365 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
366 const auto pi64_type =
367 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
371 const auto target_col_ti = window_func->getArgs().front()->get_type_info();
372 const auto target_col_size = target_col_ti.get_size();
373 const auto target_col_type_name =
376 window_func->get_type_info().get_size(), window_func->get_type_info().is_fp());
381 auto logical_null_val_lv =
383 auto target_col_null_val_lv =
385 size_t target_col_size_in_byte = target_col_size * 8;
387 target_col_ti.is_fp()
388 ?
get_fp_type(target_col_size_in_byte, cgen_state_->context_)
389 :
get_int_type(target_col_size_in_byte, cgen_state_->context_);
390 auto col_buf_type = llvm::PointerType::get(col_buf_ptr_type, 0);
391 auto target_col_buf_ptr_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
392 window_func_context->getColumnBufferForWindowFunctionExpressions().front()));
393 auto target_col_buf_lv =
394 cgen_state_->ir_builder_.CreateIntToPtr(target_col_buf_ptr_lv, col_buf_type);
397 auto partition_start_offset_buf_lv = cgen_state_->llInt(
398 reinterpret_cast<int64_t>(window_func_context->partitionStartOffset()));
399 auto partition_start_offset_ptr_lv =
400 cgen_state_->ir_builder_.CreateIntToPtr(partition_start_offset_buf_lv, pi64_type);
403 auto current_partition_start_offset_ptr_lv =
404 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
405 partition_start_offset_ptr_lv,
407 auto current_partition_start_offset_lv = cgen_state_->ir_builder_.CreateLoad(
408 current_partition_start_offset_ptr_lv->getType()->getPointerElementType(),
409 current_partition_start_offset_ptr_lv);
412 const auto partition_rowid_buf_lv =
413 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->payload()));
414 const auto partition_rowid_ptr_lv =
415 cgen_state_->ir_builder_.CreateIntToPtr(partition_rowid_buf_lv, pi32_type);
416 auto target_partition_rowid_ptr_lv =
417 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
418 partition_rowid_ptr_lv,
419 current_partition_start_offset_lv);
422 const auto sorted_rowid_lv = cgen_state_->llInt(
423 reinterpret_cast<int64_t>(window_func_context->sortedPartition()));
424 const auto sorted_rowid_ptr_lv =
425 cgen_state_->ir_builder_.CreateIntToPtr(sorted_rowid_lv, pi64_type);
426 auto target_partition_sorted_rowid_ptr_lv =
427 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
429 current_partition_start_offset_lv);
432 const auto partition_count_buf =
433 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->counts()));
434 auto partition_count_buf_ptr_lv =
435 cgen_state_->ir_builder_.CreateIntToPtr(partition_count_buf, pi32_type);
438 const auto num_elem_current_partition_ptr =
439 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
440 partition_count_buf_ptr_lv,
442 auto num_elem_current_partition_lv = cgen_state_->castToTypeIn(
443 cgen_state_->ir_builder_.CreateLoad(
444 num_elem_current_partition_ptr->getType()->getPointerElementType(),
445 num_elem_current_partition_ptr),
448 const auto order_key_ti = window_func->getOrderKeys().front()->get_type_info();
449 const auto order_key_size = order_key_ti.get_size();
452 window_func_context->getOrderKeyColumnBufferTypes().front().is_fp());
453 size_t order_key_size_in_byte = order_key_size * 8;
455 const auto order_key_buf_type = llvm::PointerType::get(
456 get_int_type(order_key_size_in_byte, cgen_state_->context_), 0);
457 const auto order_key_buf = cgen_state_->llInt(
458 reinterpret_cast<int64_t>(window_func_context->getOrderKeyColumnBuffers().front()));
459 auto order_key_buf_ptr_lv =
460 cgen_state_->ir_builder_.CreateIntToPtr(order_key_buf, order_key_buf_type);
463 const auto order_key_buf_ti =
464 window_func_context->getOrderKeyColumnBufferTypes().front();
465 auto order_key_col_null_val_lv =
469 const auto null_start_pos_buf = cgen_state_->llInt(
470 reinterpret_cast<int64_t>(window_func_context->getNullValueStartPos()));
471 const auto null_start_pos_buf_ptr =
472 cgen_state_->ir_builder_.CreateIntToPtr(null_start_pos_buf, pi64_type);
473 const auto null_start_pos_ptr =
474 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
475 null_start_pos_buf_ptr,
477 auto null_start_pos_lv = cgen_state_->ir_builder_.CreateLoad(
478 null_start_pos_ptr->getType()->getPointerElementType(),
481 const auto null_end_pos_buf = cgen_state_->llInt(
482 reinterpret_cast<int64_t>(window_func_context->getNullValueEndPos()));
483 const auto null_end_pos_buf_ptr =
484 cgen_state_->ir_builder_.CreateIntToPtr(null_end_pos_buf, pi64_type);
485 const auto null_end_pos_ptr = cgen_state_->ir_builder_.CreateGEP(
486 get_int_type(64, cgen_state_->context_), null_end_pos_buf_ptr, partition_index_lv);
487 auto null_end_pos_lv = cgen_state_->ir_builder_.CreateLoad(
488 null_end_pos_ptr->getType()->getPointerElementType(),
492 std::string compute_row_idx_on_frame_func_name =
"compute_";
493 compute_row_idx_on_frame_func_name += order_col_type_name +
"_current_row_idx_in_frame";
494 auto cur_row_idx_in_frame_lv =
495 cgen_state_->emitCall(compute_row_idx_on_frame_func_name,
496 {num_elem_current_partition_lv,
498 order_key_buf_ptr_lv,
499 target_partition_rowid_ptr_lv,
500 target_partition_sorted_rowid_ptr_lv,
501 order_key_col_null_val_lv,
505 llvm::Value* frame_start_bound_lv{
nullptr};
506 llvm::Value* frame_end_bound_lv{
nullptr};
507 std::tie(frame_start_bound_lv, frame_end_bound_lv) =
508 codegenWindowFrameBound(window_func_context,
511 cur_row_idx_in_frame_lv,
512 cgen_state_->llInt((int64_t)0),
513 order_key_buf_ptr_lv,
514 order_key_col_null_val_lv,
515 frame_start_bound_expr_lv,
516 frame_end_bound_expr_lv,
517 num_elem_current_partition_lv,
518 target_partition_rowid_ptr_lv,
519 target_partition_sorted_rowid_ptr_lv,
523 CHECK(frame_start_bound_lv);
524 CHECK(frame_end_bound_lv);
526 llvm::Value* modified_cur_row_idx_in_frame_lv{
nullptr};
527 if (is_lag_in_frame) {
528 modified_cur_row_idx_in_frame_lv =
529 cgen_state_->ir_builder_.CreateSub(cur_row_idx_in_frame_lv, offset_lv);
531 modified_cur_row_idx_in_frame_lv =
532 cgen_state_->ir_builder_.CreateAdd(cur_row_idx_in_frame_lv, offset_lv);
534 CHECK(modified_cur_row_idx_in_frame_lv);
536 std::string target_func_name =
"get_";
537 target_func_name += target_col_type_name +
"_value_";
538 target_func_name += target_col_logical_type_name +
"_type_";
539 target_func_name +=
"in_frame";
540 res_lv = cgen_state_->emitCall(target_func_name,
541 {modified_cur_row_idx_in_frame_lv,
542 frame_start_bound_lv,
545 target_partition_rowid_ptr_lv,
546 target_partition_sorted_rowid_ptr_lv,
548 target_col_null_val_lv});
550 res_lv = cgen_state_->emitCall(
552 {res_lv, logical_null_val_lv, cgen_state_->llInt((int64_t)
kSecsPerDay)});
566 const auto order_col_ti = window_func->
getOrderKeys().front()->get_type_info();
567 auto encode_date_col_val = [&order_col_ti,
this](llvm::Value* bound_expr_lv) {
568 if (order_col_ti.get_comp_param() == 16) {
569 return cgen_state_->emitCall(
570 "fixed_width_date_encode_noinline",
576 return cgen_state_->emitCall(
"fixed_width_date_encode_noinline",
582 llvm::Value* bound_expr_lv{
nullptr};
583 if (needs_bound_expr_codegen(frame_bound)) {
585 bound_expr_lv = bound_expr_lvs.front();
588 throw std::runtime_error(
589 "Range mode with date type ordering column is not supported yet.");
591 bound_expr_lv = encode_date_col_val(bound_expr_lv);
594 bound_expr_lv = cgen_state_->castToTypeIn(bound_expr_lv, 64);
597 bound_expr_lv = cgen_state_->llInt((int64_t)-1);
599 CHECK(bound_expr_lv);
600 return bound_expr_lv;
605 llvm::Value* current_row_pos_lv) {
606 const auto pi64_type =
607 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
609 auto partition_count_lv = cgen_state_->llInt(window_func_context->
partitionCount());
610 auto partition_num_count_buf_lv = cgen_state_->llInt(
612 auto partition_num_count_ptr_lv =
613 cgen_state_->ir_builder_.CreateIntToPtr(partition_num_count_buf_lv, pi64_type);
614 return cgen_state_->emitCall(
615 "compute_int64_t_lower_bound",
616 {partition_count_lv, current_row_pos_lv, partition_num_count_ptr_lv});
623 llvm::Value* current_row_pos_lv,
624 llvm::Value* current_partition_start_offset_lv,
625 llvm::Value* order_key_buf_ptr_lv,
626 llvm::Value* order_key_col_null_val_lv,
627 llvm::Value* frame_start_bound_expr_lv,
628 llvm::Value* frame_end_bound_expr_lv,
629 llvm::Value* num_elem_current_partition_lv,
630 llvm::Value* target_partition_rowid_ptr_lv,
631 llvm::Value* target_partition_sorted_rowid_ptr_lv,
632 llvm::Value* null_start_pos_lv,
633 llvm::Value* null_end_pos_lv,
637 std::string order_col_type_name{
""};
638 llvm::Value* current_col_value_lv{
nullptr};
639 llvm::Value* frame_start_bound_lv{
nullptr};
640 llvm::Value* frame_end_bound_lv{
nullptr};
642 if (window_func->hasRangeModeFraming()) {
644 CHECK(window_func->getOrderKeys().size() == 1UL);
646 const auto order_key_ti = window_func->getOrderKeys().front()->get_type_info();
647 const auto order_key_size = order_key_ti.get_size();
648 size_t order_key_size_in_byte = order_key_size * 8;
654 auto rowid_in_partition_lv =
656 auto current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
657 get_int_type(order_key_size_in_byte, cgen_state_->context_),
658 order_key_buf_ptr_lv,
659 rowid_in_partition_lv);
660 current_col_value_lv = cgen_state_->ir_builder_.CreateLoad(
661 current_col_value_ptr_lv->getType()->getPointerElementType(),
662 current_col_value_ptr_lv,
663 "current_col_value");
669 frame_start_bound_lv = cgen_state_->llInt((int64_t)0);
673 CHECK(frame_start_bound_expr_lv);
674 if (window_func->hasRowModeFraming()) {
675 frame_start_bound_lv = cgen_state_->emitCall(
"compute_row_mode_start_index_sub",
677 current_partition_start_offset_lv,
678 frame_start_bound_expr_lv});
680 CHECK(window_func->hasRangeModeFraming());
683 std::string lower_bound_func_name{
"compute_"};
684 lower_bound_func_name.append(order_col_type_name);
685 lower_bound_func_name.append(
"_lower_bound_from_ordered_index_for_timeinterval");
686 frame_start_bound_lv = cgen_state_->emitCall(
687 lower_bound_func_name,
688 {num_elem_current_partition_lv,
689 frame_start_bound_expr_lv,
690 order_key_buf_ptr_lv,
691 target_partition_rowid_ptr_lv,
692 target_partition_sorted_rowid_ptr_lv,
693 cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
697 std::string lower_bound_func_name{
"range_mode_"};
698 lower_bound_func_name.append(order_col_type_name);
699 lower_bound_func_name.append(
"_sub_frame_lower_bound");
700 frame_start_bound_lv =
701 cgen_state_->emitCall(lower_bound_func_name,
702 {num_elem_current_partition_lv,
703 current_col_value_lv,
704 order_key_buf_ptr_lv,
705 target_partition_rowid_ptr_lv,
706 target_partition_sorted_rowid_ptr_lv,
707 frame_start_bound_expr_lv,
708 order_key_col_null_val_lv,
715 if (window_func->hasRowModeFraming()) {
716 frame_start_bound_lv = cgen_state_->emitCall(
"compute_row_mode_start_index_sub",
718 current_partition_start_offset_lv,
719 cgen_state_->llInt(((int64_t)0))});
721 CHECK(window_func->hasRangeModeFraming());
722 std::string lower_bound_func_name{
"compute_"};
723 lower_bound_func_name.append(order_col_type_name);
724 lower_bound_func_name.append(
"_lower_bound_from_ordered_index");
725 frame_start_bound_lv = cgen_state_->emitCall(lower_bound_func_name,
726 {num_elem_current_partition_lv,
727 current_col_value_lv,
728 order_key_buf_ptr_lv,
729 target_partition_rowid_ptr_lv,
730 target_partition_sorted_rowid_ptr_lv,
731 order_key_col_null_val_lv,
738 CHECK(frame_start_bound_expr_lv);
739 if (window_func->hasRowModeFraming()) {
740 frame_start_bound_lv = cgen_state_->emitCall(
"compute_row_mode_start_index_add",
742 current_partition_start_offset_lv,
743 frame_start_bound_expr_lv,
744 num_elem_current_partition_lv});
746 CHECK(window_func->hasRangeModeFraming());
749 std::string lower_bound_func_name{
"compute_"};
750 lower_bound_func_name.append(order_col_type_name);
751 lower_bound_func_name.append(
"_lower_bound_from_ordered_index_for_timeinterval");
752 frame_start_bound_lv = cgen_state_->emitCall(
753 lower_bound_func_name,
754 {num_elem_current_partition_lv,
755 frame_start_bound_expr_lv,
756 order_key_buf_ptr_lv,
757 target_partition_rowid_ptr_lv,
758 target_partition_sorted_rowid_ptr_lv,
759 cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
763 std::string lower_bound_func_name{
"range_mode_"};
764 lower_bound_func_name.append(order_col_type_name);
765 lower_bound_func_name.append(
"_add_frame_lower_bound");
766 frame_start_bound_lv =
767 cgen_state_->emitCall(lower_bound_func_name,
768 {num_elem_current_partition_lv,
769 current_col_value_lv,
770 order_key_buf_ptr_lv,
771 target_partition_rowid_ptr_lv,
772 target_partition_sorted_rowid_ptr_lv,
773 frame_start_bound_expr_lv,
774 order_key_col_null_val_lv,
780 CHECK(
false) <<
"frame start cannot be UNBOUNDED FOLLOWING";
786 CHECK(
false) <<
"frame end cannot be UNBOUNDED PRECEDING";
789 CHECK(frame_end_bound_expr_lv);
790 if (window_func->hasRowModeFraming()) {
791 frame_end_bound_lv = cgen_state_->emitCall(
"compute_row_mode_end_index_sub",
793 current_partition_start_offset_lv,
794 frame_end_bound_expr_lv});
796 CHECK(window_func->hasRangeModeFraming());
799 std::string upper_bound_func_name{
"compute_"};
800 upper_bound_func_name.append(order_col_type_name);
801 upper_bound_func_name.append(
"_upper_bound_from_ordered_index_for_timeinterval");
802 frame_end_bound_lv = cgen_state_->emitCall(
803 upper_bound_func_name,
804 {num_elem_current_partition_lv,
805 frame_end_bound_expr_lv,
806 order_key_buf_ptr_lv,
807 target_partition_rowid_ptr_lv,
808 target_partition_sorted_rowid_ptr_lv,
809 cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
813 std::string upper_bound_func_name{
"range_mode_"};
814 upper_bound_func_name.append(order_col_type_name);
815 upper_bound_func_name.append(
"_sub_frame_upper_bound");
816 frame_end_bound_lv = cgen_state_->emitCall(upper_bound_func_name,
817 {num_elem_current_partition_lv,
818 current_col_value_lv,
819 order_key_buf_ptr_lv,
820 target_partition_rowid_ptr_lv,
821 target_partition_sorted_rowid_ptr_lv,
822 frame_end_bound_expr_lv,
823 order_key_col_null_val_lv,
830 if (window_func->hasRowModeFraming()) {
831 frame_end_bound_lv = cgen_state_->emitCall(
"compute_row_mode_end_index_sub",
833 current_partition_start_offset_lv,
834 cgen_state_->llInt((int64_t)0)});
836 CHECK(window_func->hasRangeModeFraming());
837 std::string upper_bound_func_name{
"compute_"};
838 upper_bound_func_name.append(order_col_type_name);
839 upper_bound_func_name.append(
"_upper_bound_from_ordered_index");
840 frame_end_bound_lv = cgen_state_->emitCall(upper_bound_func_name,
841 {num_elem_current_partition_lv,
842 current_col_value_lv,
843 order_key_buf_ptr_lv,
844 target_partition_rowid_ptr_lv,
845 target_partition_sorted_rowid_ptr_lv,
846 order_key_col_null_val_lv,
852 CHECK(frame_end_bound_expr_lv);
853 if (window_func->hasRowModeFraming()) {
854 frame_end_bound_lv = cgen_state_->emitCall(
"compute_row_mode_end_index_add",
856 current_partition_start_offset_lv,
857 frame_end_bound_expr_lv,
858 num_elem_current_partition_lv});
860 CHECK(window_func->hasRangeModeFraming());
863 std::string upper_bound_func_name{
"compute_"};
864 upper_bound_func_name.append(order_col_type_name);
865 upper_bound_func_name.append(
"_upper_bound_from_ordered_index_for_timeinterval");
866 frame_end_bound_lv = cgen_state_->emitCall(
867 upper_bound_func_name,
868 {num_elem_current_partition_lv,
869 frame_end_bound_expr_lv,
870 order_key_buf_ptr_lv,
871 target_partition_rowid_ptr_lv,
872 target_partition_sorted_rowid_ptr_lv,
873 cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
877 std::string upper_bound_func_name{
"range_mode_"};
878 upper_bound_func_name.append(order_col_type_name);
879 upper_bound_func_name.append(
"_add_frame_upper_bound");
880 frame_end_bound_lv = cgen_state_->emitCall(upper_bound_func_name,
881 {num_elem_current_partition_lv,
882 current_col_value_lv,
883 order_key_buf_ptr_lv,
884 target_partition_rowid_ptr_lv,
885 target_partition_sorted_rowid_ptr_lv,
886 frame_end_bound_expr_lv,
887 order_key_col_null_val_lv,
896 frame_end_bound_lv = num_elem_current_partition_lv;
898 CHECK(frame_start_bound_lv);
899 CHECK(frame_end_bound_lv);
900 return std::make_pair(frame_start_bound_lv, frame_end_bound_lv);
906 const auto window_func_context =
908 const auto window_func = window_func_context->getWindowFunction();
910 const auto window_func_null_val =
911 window_func_ti.is_fp()
912 ? cgen_state_->inlineFpNull(window_func_ti)
913 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
914 const auto&
args = window_func->getArgs();
915 llvm::Value* crt_val;
919 crt_val = cgen_state_->llInt(int64_t(1));
921 const auto arg_lvs = code_generator.
codegen(
args.front().get(),
true, co);
922 CHECK_EQ(arg_lvs.size(), size_t(1));
925 arg_lvs.front(),
args.front()->get_type_info(), window_func_ti,
false);
927 crt_val = window_func_ti.get_type() ==
kFLOAT
929 : cgen_state_->castToTypeIn(arg_lvs.front(), 64);
932 if (window_func_context->needsToBuildAggregateTree()) {
939 const auto pi64_type =
940 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
941 const auto ppi64_type = llvm::PointerType::get(pi64_type, 0);
942 const auto pi32_type =
943 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
946 const auto frame_start_bound = window_func->getFrameStartBound();
947 const auto frame_end_bound = window_func->getFrameEndBound();
948 auto frame_start_bound_expr_lv =
949 codegenFrameBoundExpr(window_func, frame_start_bound, code_generator, co);
950 auto frame_end_bound_expr_lv =
951 codegenFrameBoundExpr(window_func, frame_end_bound, code_generator, co);
952 CHECK(frame_start_bound_expr_lv);
953 CHECK(frame_end_bound_expr_lv);
956 auto current_row_pos_lv = code_generator.
posArg(
nullptr);
957 auto partition_index_lv =
958 codegenCurrentPartitionIndex(window_func_context, current_row_pos_lv);
961 const auto target_col_ti = window_func->getArgs().front()->get_type_info();
962 const auto target_col_size = target_col_ti.get_size();
963 const auto col_type_name =
967 auto partition_start_offset_buf_lv = cgen_state_->llInt(
968 reinterpret_cast<int64_t>(window_func_context->partitionStartOffset()));
969 auto partition_start_offset_ptr_lv =
970 cgen_state_->ir_builder_.CreateIntToPtr(partition_start_offset_buf_lv, pi64_type);
973 auto current_partition_start_offset_ptr_lv =
974 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
975 partition_start_offset_ptr_lv,
977 auto current_partition_start_offset_lv = cgen_state_->ir_builder_.CreateLoad(
978 current_partition_start_offset_ptr_lv->getType()->getPointerElementType(),
979 current_partition_start_offset_ptr_lv);
982 const auto partition_rowid_buf_lv =
983 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->payload()));
984 const auto partition_rowid_ptr_lv =
985 cgen_state_->ir_builder_.CreateIntToPtr(partition_rowid_buf_lv, pi32_type);
986 auto target_partition_rowid_ptr_lv =
987 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
988 partition_rowid_ptr_lv,
989 current_partition_start_offset_lv);
992 const auto sorted_rowid_lv = cgen_state_->llInt(
993 reinterpret_cast<int64_t>(window_func_context->sortedPartition()));
994 const auto sorted_rowid_ptr_lv =
995 cgen_state_->ir_builder_.CreateIntToPtr(sorted_rowid_lv, pi64_type);
996 auto target_partition_sorted_rowid_ptr_lv =
997 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
999 current_partition_start_offset_lv);
1002 const auto partition_count_buf =
1003 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->counts()));
1004 auto partition_count_buf_ptr_lv =
1005 cgen_state_->ir_builder_.CreateIntToPtr(partition_count_buf, pi32_type);
1008 const auto num_elem_current_partition_ptr =
1009 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
1010 partition_count_buf_ptr_lv,
1011 partition_index_lv);
1012 auto num_elem_current_partition_lv = cgen_state_->castToTypeIn(
1013 cgen_state_->ir_builder_.CreateLoad(
1014 num_elem_current_partition_ptr->getType()->getPointerElementType(),
1015 num_elem_current_partition_ptr),
1018 const auto order_key_ti = window_func->getOrderKeys().front()->get_type_info();
1019 const auto order_key_size = order_key_ti.get_size();
1022 window_func_context->getOrderKeyColumnBufferTypes().front().is_fp());
1023 size_t order_key_size_in_byte = order_key_size * 8;
1025 const auto order_key_buf_type = llvm::PointerType::get(
1026 get_int_type(order_key_size_in_byte, cgen_state_->context_), 0);
1027 const auto order_key_buf = cgen_state_->llInt(reinterpret_cast<int64_t>(
1028 window_func_context->getOrderKeyColumnBuffers().front()));
1029 auto order_key_buf_ptr_lv =
1030 cgen_state_->ir_builder_.CreateIntToPtr(order_key_buf, order_key_buf_type);
1033 const auto order_key_buf_ti =
1034 window_func_context->getOrderKeyColumnBufferTypes().front();
1035 llvm::Value* order_key_col_null_val_lv{
nullptr};
1036 switch (order_key_buf_ti.get_type()) {
1040 switch (order_key_buf_ti.get_size()) {
1042 order_key_col_null_val_lv =
1047 order_key_col_null_val_lv =
1052 order_key_col_null_val_lv =
1057 order_key_col_null_val_lv =
1067 order_key_col_null_val_lv = cgen_state_->inlineNull(order_key_buf_ti);
1073 const auto null_start_pos_buf = cgen_state_->llInt(
1074 reinterpret_cast<int64_t>(window_func_context->getNullValueStartPos()));
1075 const auto null_start_pos_buf_ptr =
1076 cgen_state_->ir_builder_.CreateIntToPtr(null_start_pos_buf, pi64_type);
1077 const auto null_start_pos_ptr =
1078 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
1079 null_start_pos_buf_ptr,
1080 partition_index_lv);
1081 auto null_start_pos_lv = cgen_state_->ir_builder_.CreateLoad(
1082 null_start_pos_ptr->getType()->getPointerElementType(),
1085 const auto null_end_pos_buf = cgen_state_->llInt(
1086 reinterpret_cast<int64_t>(window_func_context->getNullValueEndPos()));
1087 const auto null_end_pos_buf_ptr =
1088 cgen_state_->ir_builder_.CreateIntToPtr(null_end_pos_buf, pi64_type);
1089 const auto null_end_pos_ptr =
1090 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
1091 null_end_pos_buf_ptr,
1092 partition_index_lv);
1093 auto null_end_pos_lv = cgen_state_->ir_builder_.CreateLoad(
1094 null_end_pos_ptr->getType()->getPointerElementType(),
1098 llvm::Value* frame_start_bound_lv{
nullptr};
1099 llvm::Value* frame_end_bound_lv{
nullptr};
1100 std::tie(frame_start_bound_lv, frame_end_bound_lv) =
1101 codegenWindowFrameBound(window_func_context,
1105 current_partition_start_offset_lv,
1106 order_key_buf_ptr_lv,
1107 order_key_col_null_val_lv,
1108 frame_start_bound_expr_lv,
1109 frame_end_bound_expr_lv,
1110 num_elem_current_partition_lv,
1111 target_partition_rowid_ptr_lv,
1112 target_partition_sorted_rowid_ptr_lv,
1116 CHECK(frame_start_bound_lv);
1117 CHECK(frame_end_bound_lv);
1120 llvm::Value* aggregation_trees_lv{
nullptr};
1121 llvm::Value* invalid_val_lv{
nullptr};
1122 llvm::Value* null_val_lv{
nullptr};
1123 std::string aggregation_tree_search_func_name{
"search_"};
1124 std::string aggregation_tree_getter_func_name{
"get_"};
1128 auto agg_expr_ti =
args.front()->get_type_info();
1129 switch (agg_expr_ti.get_type()) {
1137 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::max());
1139 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::lowest());
1141 invalid_val_lv = cgen_state_->llInt((int64_t)0);
1143 null_val_lv = cgen_state_->llInt(inline_int_null_value<int64_t>());
1144 aggregation_tree_search_func_name +=
"int64_t";
1145 aggregation_tree_getter_func_name +=
"integer";
1151 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::max());
1153 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::lowest());
1155 invalid_val_lv = cgen_state_->llFp((
double)0);
1158 aggregation_tree_search_func_name +=
"double";
1159 aggregation_tree_getter_func_name +=
"double";
1170 aggregation_tree_search_func_name +=
"_derived";
1171 aggregation_tree_getter_func_name +=
"_derived";
1175 if (agg_expr_ti.is_integer() || agg_expr_ti.is_decimal()) {
1177 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1178 window_func_context->getDerivedAggregationTreesForIntegerTypeWindowExpr()));
1180 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1181 window_func_context->getAggregationTreesForIntegerTypeWindowExpr()));
1183 }
else if (agg_expr_ti.is_fp()) {
1185 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1186 window_func_context->getDerivedAggregationTreesForDoubleTypeWindowExpr()));
1188 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1189 window_func_context->getAggregationTreesForDoubleTypeWindowExpr()));
1193 CHECK(aggregation_trees_lv);
1194 CHECK(invalid_val_lv);
1195 aggregation_tree_search_func_name +=
"_aggregation_tree";
1196 aggregation_tree_getter_func_name +=
"_aggregation_tree";
1199 auto aggregation_trees_ptr =
1200 cgen_state_->ir_builder_.CreateIntToPtr(aggregation_trees_lv, ppi64_type);
1201 auto target_aggregation_tree_lv = cgen_state_->emitCall(
1202 aggregation_tree_getter_func_name, {aggregation_trees_ptr, partition_index_lv});
1205 const auto tree_depth_buf = cgen_state_->llInt(
1206 reinterpret_cast<int64_t>(window_func_context->getAggregateTreeDepth()));
1207 const auto tree_depth_buf_ptr =
1208 cgen_state_->ir_builder_.CreateIntToPtr(tree_depth_buf, pi64_type);
1209 const auto current_partition_tree_depth_buf_ptr = cgen_state_->ir_builder_.CreateGEP(
1210 get_int_type(64, cgen_state_->context_), tree_depth_buf_ptr, partition_index_lv);
1211 const auto current_partition_tree_depth_lv = cgen_state_->ir_builder_.CreateLoad(
1212 current_partition_tree_depth_buf_ptr->getType()->getPointerElementType(),
1213 current_partition_tree_depth_buf_ptr);
1216 const auto aggregation_tree_fanout_lv = cgen_state_->llInt(
1217 static_cast<int64_t>(window_func_context->getAggregateTreeFanout()));
1220 const auto agg_type_lv =
1221 cgen_state_->llInt(static_cast<int32_t>(window_func->getKind()));
1226 cgen_state_->emitCall(aggregation_tree_search_func_name,
1227 {target_aggregation_tree_lv,
1228 frame_start_bound_lv,
1230 current_partition_tree_depth_lv,
1231 aggregation_tree_fanout_lv,
1232 cgen_state_->llBool(agg_expr_ti.is_decimal()),
1233 cgen_state_->llInt((int64_t)agg_expr_ti.get_scale()),
1239 std::string null_handler_func_name{
"handle_null_val_"};
1240 std::vector<llvm::Value*> null_handler_args{res_lv, null_val_lv};
1246 if (agg_expr_ti.is_fp()) {
1248 null_handler_func_name +=
"double_double";
1251 null_handler_func_name +=
"double_int64_t";
1253 }
else if (agg_expr_ti.is_fp()) {
1255 null_handler_func_name +=
"double_double";
1258 null_handler_func_name +=
"int64_t_int64_t";
1260 null_handler_func_name +=
"_window_framing_agg";
1264 if (agg_expr_ti.is_fp()) {
1265 null_handler_args.push_back(cgen_state_->llFp((
double)0));
1267 null_handler_args.push_back(cgen_state_->llInt((int64_t)0));
1272 null_handler_args.push_back(cgen_state_->castToTypeIn(window_func_null_val, 64));
1274 res_lv = cgen_state_->emitCall(null_handler_func_name, null_handler_args);
1279 return cgen_state_->ir_builder_.CreateFPToSI(
1284 llvm::Value* multiplicity_lv =
nullptr;
1287 cgen_state_->emitCall(agg_name, {aggregate_state, crt_val});
1289 cgen_state_->emitCall(agg_name +
"_skip_val",
1290 {aggregate_state, crt_val, window_func_null_val});
1293 codegenWindowAvgEpilogue(crt_val, window_func_null_val, multiplicity_lv);
1295 return codegenAggregateWindowState();
1300 llvm::Value* window_func_null_val,
1301 llvm::Value* multiplicity_lv) {
1303 const auto window_func_context =
1305 const auto window_func = window_func_context->getWindowFunction();
1307 const auto pi32_type =
1308 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1309 const auto pi64_type =
1310 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1311 const auto aggregate_state_type =
1312 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1313 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1314 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1315 auto aggregate_state_count = cgen_state_->ir_builder_.CreateIntToPtr(
1316 aggregate_state_count_i64, aggregate_state_type);
1317 std::string agg_count_func_name =
"agg_count";
1318 switch (window_func_ti.get_type()) {
1320 agg_count_func_name +=
"_float";
1324 agg_count_func_name +=
"_double";
1331 agg_count_func_name +=
"_skip_val";
1332 cgen_state_->emitCall(agg_count_func_name,
1333 {aggregate_state_count, crt_val, window_func_null_val});
1338 const auto pi32_type =
1339 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1340 const auto pi64_type =
1341 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1342 const auto window_func_context =
1346 const auto aggregate_state_type =
1347 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1348 auto aggregate_state = aggregateWindowStatePtr();
1350 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1351 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1352 auto aggregate_state_count = cgen_state_->ir_builder_.CreateIntToPtr(
1353 aggregate_state_count_i64, aggregate_state_type);
1355 switch (window_func_ti.get_type()) {
1357 return cgen_state_->emitCall(
1358 "load_avg_float", {aggregate_state, aggregate_state_count, double_null_lv});
1361 return cgen_state_->emitCall(
1362 "load_avg_double", {aggregate_state, aggregate_state_count, double_null_lv});
1365 return cgen_state_->emitCall(
1368 aggregate_state_count,
1370 cgen_state_->llInt<int32_t>(window_func_ti.get_scale())});
1373 return cgen_state_->emitCall(
1374 "load_avg_int", {aggregate_state, aggregate_state_count, double_null_lv});
1379 return cgen_state_->ir_builder_.CreateLoad(
1380 aggregate_state->getType()->getPointerElementType(), aggregate_state);
1382 switch (window_func_ti.get_type()) {
1384 return cgen_state_->emitCall(
"load_float", {aggregate_state});
1387 return cgen_state_->emitCall(
"load_double", {aggregate_state});
1390 return cgen_state_->ir_builder_.CreateLoad(
1391 aggregate_state->getType()->getPointerElementType(), aggregate_state);
std::string get_col_type_name_by_size(const size_t size, const bool is_fp)
bool hasRangeModeFraming() const
llvm::Value * castToTypeIn(llvm::Value *val, const size_t bit_width)
SqlWindowFunctionKind getKind() const
HOST DEVICE int get_size() const
llvm::Value * codegenCurrentPartitionIndex(const WindowFunctionContext *window_func_context, llvm::Value *current_row_pos_lv)
bool is_timestamp() const
const std::vector< SQLTypeInfo > & getOrderKeyColumnBufferTypes() const
llvm::Value * posArg(const Analyzer::Expr *) const
llvm::Value * aggregateWindowStatePtr()
llvm::Value * codegenWindowFunctionOnFrame(const CompilationOptions &co)
llvm::Type * get_fp_type(const int width, llvm::LLVMContext &context)
HOST DEVICE SQLTypes get_type() const
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getOrderKeys() const
static WindowFunctionContext * getActiveWindowFunctionContext(Executor *executor)
std::string get_window_agg_name(const SqlWindowFunctionKind kind, const SQLTypeInfo &window_func_ti)
void codegenWindowAvgEpilogue(llvm::Value *crt_val, llvm::Value *window_func_null_val, llvm::Value *multiplicity_lv)
size_t partitionCount() const
SqlWindowFrameBoundType getBoundType() const
llvm::Value * codegenWindowPosition(const WindowFunctionContext *window_func_context, llvm::Value *pos_arg)
static const WindowProjectNodeContext * get(Executor *executor)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getArgs() const
int get_logical_size() const
const WindowFunctionContext * activateWindowFunctionContext(Executor *executor, const size_t target_index) const
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
const Analyzer::Expr * getBoundExpr() const
llvm::ConstantFP * llFp(const float v) const
const SQLTypeInfo & get_type_info() const
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
HOST DEVICE EncodingType get_compression() const
std::pair< llvm::Value *, llvm::Value * > codegenWindowFrameBound(WindowFunctionContext *window_func_context, const Analyzer::WindowFrame *frame_start_bound, const Analyzer::WindowFrame *frame_end_bound, llvm::Value *current_row_pos_lv, llvm::Value *current_partition_start_offset_lv, llvm::Value *order_key_buf_ptr_lv, llvm::Value *order_key_col_null_val_lv, llvm::Value *frame_start_bound_expr_lv, llvm::Value *frame_end_bound_expr_lv, llvm::Value *num_elem_current_partition_lv, llvm::Value *target_partition_rowid_ptr_lv, llvm::Value *target_partition_sorted_rowid_ptr_lv, llvm::Value *null_start_pos_lv, llvm::Value *null_end_pos_lv, CodeGenerator &code_generator)
void codegenWindowFunctionStateInit(llvm::Value *aggregate_state)
constexpr float inline_fp_null_value< float >()
const std::vector< const int8_t * > & getOrderKeyColumnBuffers() const
constexpr double inline_fp_null_value< double >()
const int64_t * partitionNumCountBuf() const
llvm::Value * toBool(llvm::Value *)
llvm::Value * codegenAggregateWindowState()
llvm::Value * codegenFrameBoundExpr(const Analyzer::WindowFunction *window_func, const Analyzer::WindowFrame *frame_bound, CodeGenerator &code_generator, const CompilationOptions &co)
llvm::ConstantInt * llInt(const T v) const
llvm::Value * codegenWindowFunctionAggregate(const CompilationOptions &co)
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
const Analyzer::WindowFunction * getWindowFunction() const
bool is_dict_encoded_string() const
llvm::Value * get_null_value_by_size(CgenState *cgen_state, SQLTypeInfo col_ti)
llvm::Value * codegenWindowFunctionAggregateCalls(llvm::Value *aggregate_state, const CompilationOptions &co)
llvm::Value * codegenWindowFunction(const size_t target_index, const CompilationOptions &co)
llvm::BasicBlock * codegenWindowResetStateControlFlow()
llvm::Value * get_null_value_by_size_with_encoding(CgenState *cgen_state, SQLTypeInfo col_ti)
SQLTypeInfo get_adjusted_window_type_info(const Analyzer::WindowFunction *window_func)