29 #include <llvm/Bitcode/BitcodeReader.h> 30 #include <llvm/IR/Function.h> 31 #include <llvm/IR/IRBuilder.h> 32 #include <llvm/IR/Verifier.h> 33 #include <llvm/Support/SourceMgr.h> 34 #include <llvm/Support/raw_os_ostream.h> 51 return function->add<
Load>(
53 ptr->
label() +
"_loaded");
77 LOG(
FATAL) <<
"Invalid byte width: " << compact_sz;
88 const size_t chosen_bytes,
90 Function* ir_reduce_one_entry) {
92 const auto dest_name = agg_kind +
"_dest";
93 if (sql_type.is_fp()) {
94 if (chosen_bytes ==
sizeof(
float)) {
95 const auto agg = ir_reduce_one_entry->add<
Cast>(
98 ir_reduce_one_entry->add<
Call>(
99 "agg_" + agg_kind +
"_float", std::vector<const Value*>{agg, val},
"");
101 CHECK_EQ(chosen_bytes,
sizeof(
double));
102 const auto agg = ir_reduce_one_entry->add<
Cast>(
105 ir_reduce_one_entry->add<
Call>(
106 "agg_" + agg_kind +
"_double", std::vector<const Value*>{agg, val},
"");
109 if (chosen_bytes ==
sizeof(int32_t)) {
110 const auto agg = ir_reduce_one_entry->add<
Cast>(
112 const auto val =
emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
113 ir_reduce_one_entry->add<
Call>(
114 "agg_" + agg_kind +
"_int32", std::vector<const Value*>{agg, val},
"");
116 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
117 const auto agg = ir_reduce_one_entry->add<
Cast>(
119 const auto val =
emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
120 ir_reduce_one_entry->add<
Call>(
121 "agg_" + agg_kind, std::vector<const Value*>{agg, val},
"");
130 const int64_t init_val,
131 const size_t chosen_bytes,
133 Function* ir_reduce_one_entry) {
134 const auto dest_name = agg_kind +
"_dest";
137 if (sql_type.is_fp()) {
138 if (chosen_bytes ==
sizeof(
float)) {
139 const auto agg = ir_reduce_one_entry->add<
Cast>(
142 const auto init_val_lv = ir_reduce_one_entry->addConstant<
ConstantFP>(
143 *
reinterpret_cast<const float*
>(may_alias_ptr(&init_val)),
Type::Float);
144 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_float_skip_val",
145 std::vector<const Value*>{agg, val, init_val_lv},
148 CHECK_EQ(chosen_bytes,
sizeof(
double));
149 const auto agg = ir_reduce_one_entry->add<
Cast>(
152 const auto init_val_lv = ir_reduce_one_entry->addConstant<
ConstantFP>(
153 *
reinterpret_cast<const double*
>(may_alias_ptr(&init_val)),
Type::Double);
154 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_double_skip_val",
155 std::vector<const Value*>{agg, val, init_val_lv},
159 if (chosen_bytes ==
sizeof(int32_t)) {
160 const auto agg = ir_reduce_one_entry->add<
Cast>(
162 const auto val =
emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
163 const auto init_val_lv =
165 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_int32_skip_val",
166 std::vector<const Value*>{agg, val, init_val_lv},
169 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
170 const auto agg = ir_reduce_one_entry->add<
Cast>(
172 const auto val =
emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
173 const auto init_val_lv =
175 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_skip_val",
176 std::vector<const Value*>{agg, val, init_val_lv},
182 agg_kind, val_ptr, other_ptr, chosen_bytes, agg_info, ir_reduce_one_entry);
189 const size_t chosen_bytes,
190 Function* ir_reduce_one_entry) {
191 const auto dest_name =
"count_dest";
192 if (chosen_bytes ==
sizeof(int32_t)) {
193 const auto agg = ir_reduce_one_entry->add<
Cast>(
195 const auto val =
emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
196 ir_reduce_one_entry->add<
Call>(
197 "agg_sum_int32", std::vector<const Value*>{agg, val},
"");
199 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
200 const auto agg = ir_reduce_one_entry->add<
Cast>(
202 const auto val =
emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
203 ir_reduce_one_entry->add<
Call>(
"agg_sum", std::vector<const Value*>{agg, val},
"");
212 const int64_t init_val,
213 const size_t chosen_bytes,
214 Function* ir_reduce_one_entry) {
215 const auto func_name =
"write_projection_int" +
std::to_string(chosen_bytes * 8);
216 if (chosen_bytes ==
sizeof(int32_t)) {
217 const auto proj_val =
emit_load_i32(other_pi8, ir_reduce_one_entry);
218 ir_reduce_one_entry->add<
Call>(
220 std::vector<const Value*>{
226 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
227 const auto proj_val =
emit_load_i64(other_pi8, ir_reduce_one_entry);
228 ir_reduce_one_entry->add<
Call>(
230 std::vector<const Value*>{
243 const int64_t init_val,
244 const size_t chosen_bytes,
245 Function* ir_reduce_one_entry) {
246 if (chosen_bytes ==
sizeof(int32_t)) {
247 const auto func_name =
"checked_single_agg_id_int32";
248 const auto proj_val =
emit_load_i32(other_pi8, ir_reduce_one_entry);
249 const auto slot_pi32 = ir_reduce_one_entry->add<
Cast>(
251 return ir_reduce_one_entry->add<
Call>(
254 std::vector<const Value*>{
260 const auto func_name =
"checked_single_agg_id";
261 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
262 const auto proj_val =
emit_load_i64(other_pi8, ir_reduce_one_entry);
263 const auto slot_pi64 = ir_reduce_one_entry->add<
Cast>(
266 return ir_reduce_one_entry->add<
Call>(
269 std::vector<const Value*>{
278 const std::string
name,
279 const std::vector<Function::NamedArg>& arg_types,
281 const bool always_inline) {
282 return std::make_unique<Function>(
name, arg_types, ret_type, always_inline);
295 std::string this_ptr_name;
296 std::string that_ptr_name;
299 this_ptr_name =
"this_targets_ptr";
300 that_ptr_name =
"that_targets_ptr";
305 this_ptr_name =
"this_row_ptr";
306 that_ptr_name =
"that_row_ptr";
310 LOG(
FATAL) <<
"Unexpected query description type";
315 {that_ptr_name, Type::Int8Ptr},
356 std::vector<llvm::Type*> parameter_types;
357 const auto& arg_types =
function->arg_types();
358 for (
const auto& named_arg : arg_types) {
360 parameter_types.push_back(
llvm_type(named_arg.type, ctx));
362 const auto func_type = llvm::FunctionType::get(
363 llvm_type(function->ret_type(), ctx), parameter_types,
false);
364 const auto linkage =
function->always_inline() ? llvm::Function::PrivateLinkage
365 : llvm::Function::ExternalLinkage;
367 llvm::Function::Create(func_type, linkage, function->name(), cgen_state->
module_);
368 const auto arg_it = func->arg_begin();
369 for (
size_t i = 0; i < arg_types.size(); ++i) {
370 const auto arg = &*(arg_it + i);
371 arg->setName(arg_types[i].
name);
373 if (function->always_inline()) {
387 return reduction_code;
399 const int8_t* that_ptr1,
400 const int8_t* that_ptr2,
401 const int64_t init_val) {
402 const auto rhs_proj_col = *
reinterpret_cast<const int64_t*
>(that_ptr1);
403 if (rhs_proj_col != init_val) {
404 *
reinterpret_cast<int64_t*
>(this_ptr1) = rhs_proj_col;
406 CHECK(this_ptr2 && that_ptr2);
407 *
reinterpret_cast<int64_t*
>(this_ptr2) = *reinterpret_cast<const int64_t*>(that_ptr2);
413 const void* serialized_varlen_buffer_handle,
416 const int8_t* that_ptr1,
417 const int8_t* that_ptr2,
418 const int64_t init_val,
419 const int64_t length_to_elems) {
420 if (!serialized_varlen_buffer_handle) {
424 const auto& serialized_varlen_buffer =
425 *
reinterpret_cast<const std::vector<std::string>*
>(serialized_varlen_buffer_handle);
426 if (!serialized_varlen_buffer.empty()) {
427 const auto rhs_proj_col = *
reinterpret_cast<const int64_t*
>(that_ptr1);
428 CHECK_LT(static_cast<size_t>(rhs_proj_col), serialized_varlen_buffer.size());
429 const auto& varlen_bytes_str = serialized_varlen_buffer[rhs_proj_col];
430 const auto str_ptr =
reinterpret_cast<const int8_t*
>(varlen_bytes_str.c_str());
431 *
reinterpret_cast<int64_t*
>(this_ptr1) = reinterpret_cast<const int64_t>(str_ptr);
432 *
reinterpret_cast<int64_t*
>(this_ptr2) =
433 static_cast<int64_t>(varlen_bytes_str.size() / length_to_elems);
443 const int64_t old_set_handle,
444 const void* that_qmd_handle,
445 const void* this_qmd_handle,
446 const int64_t target_logical_idx) {
449 const auto& new_count_distinct_desc =
451 const auto& old_count_distinct_desc =
452 this_qmd->getCountDistinctDescriptor(target_logical_idx);
454 CHECK(old_count_distinct_desc.impl_type_ == new_count_distinct_desc.impl_type_);
456 new_set_handle, old_set_handle, new_count_distinct_desc, old_count_distinct_desc);
460 const int64_t old_set_handle,
461 const void* that_qmd_handle,
462 const void* this_qmd_handle,
463 const int64_t target_logical_idx) {
471 const uint32_t key_count,
472 const void* this_qmd_handle,
473 const int8_t* that_buff,
474 const uint32_t that_entry_idx,
475 const uint32_t that_entry_count,
476 const uint32_t row_size_bytes,
482 this_qmd.getEntryCount(),
483 reinterpret_cast<const int64_t*
>(key),
485 this_qmd.getEffectiveKeyWidth(),
487 reinterpret_cast<const int64_t*
>(that_buff),
490 row_size_bytes >> 3);
491 *buff_out = gvi.first;
504 const std::vector<TargetInfo>& targets,
505 const std::vector<int64_t>& target_init_vals)
506 : query_mem_desc_(query_mem_desc)
508 , target_init_vals_(target_init_vals) {}
561 LOG(
FATAL) <<
"Unexpected query description type";
568 return reduction_code;
573 if (compilation_context) {
581 std::move(reduction_code.ir_is_empty),
582 std::move(reduction_code.ir_reduce_one_entry),
583 std::move(reduction_code.ir_reduce_one_entry_idx),
584 std::move(reduction_code.ir_reduce_loop)};
586 reduction_code.cgen_state.reset(
new CgenState({},
false));
587 auto cgen_state = reduction_code.cgen_state.get();
589 cgen_state->module_ = module.get();
592 auto ir_reduce_one_entry =
594 auto ir_reduce_one_entry_idx =
596 auto ir_reduce_loop =
598 std::unordered_map<const Function*, llvm::Function*> f;
599 f.emplace(reduction_code.ir_is_empty.get(), ir_is_empty);
600 f.emplace(reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry);
601 f.emplace(reduction_code.ir_reduce_one_entry_idx.get(), ir_reduce_one_entry_idx);
602 f.emplace(reduction_code.ir_reduce_loop.get(), ir_reduce_loop);
605 reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code, f);
607 ir_reduce_one_entry_idx,
611 reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code, f);
612 reduction_code.llvm_reduce_loop = ir_reduce_loop;
613 reduction_code.module = std::move(module);
618 ir_reduce_one_entry_idx,
631 auto ir_is_empty = reduction_code.
ir_is_empty.get();
635 Value* empty_key_val{
nullptr};
636 const auto keys_ptr = ir_is_empty->arg(0);
648 "is_empty_slot_ptr");
649 const auto compact_sz =
652 empty_key_val = ir_is_empty->addConstant<
ConstantInt>(
674 ir_is_empty->add<
Ret>(ret);
680 const auto this_row_ptr = ir_reduce_one_entry->arg(0);
681 const auto that_row_ptr = ir_reduce_one_entry->arg(1);
682 const auto that_is_empty =
684 std::vector<const Value*>{that_row_ptr},
691 ir_reduce_one_entry->add<
MemCpy>(
698 const auto key_bytes_lv =
700 const auto this_targets_start_ptr = ir_reduce_one_entry->add<
GetElementPtr>(
701 this_row_ptr, key_bytes_lv,
"this_targets_start");
702 const auto that_targets_start_ptr = ir_reduce_one_entry->add<
GetElementPtr>(
703 that_row_ptr, key_bytes_lv,
"that_targets_start");
706 ir_reduce_one_entry, this_targets_start_ptr, that_targets_start_ptr);
710 Function* ir_reduce_one_entry,
711 Value* this_targets_start_ptr,
712 Value* that_targets_start_ptr)
const {
714 Value* this_targets_ptr = this_targets_start_ptr;
715 Value* that_targets_ptr = that_targets_start_ptr;
716 size_t init_agg_val_idx = 0;
717 for (
size_t target_logical_idx = 0; target_logical_idx <
targets_.size();
718 ++target_logical_idx) {
719 const auto& target_info =
targets_[target_logical_idx];
720 const auto& slots_for_col = col_slot_context.getSlotsForCol(target_logical_idx);
721 Value* this_ptr2{
nullptr};
722 Value* that_ptr2{
nullptr};
724 bool two_slot_target{
false};
725 if (target_info.is_agg &&
726 (target_info.agg_kind ==
kAVG ||
727 (target_info.agg_kind ==
kSAMPLE && target_info.sql_type.is_varlen()))) {
731 two_slot_target =
true;
734 for (
size_t target_slot_idx = slots_for_col.front();
735 target_slot_idx < slots_for_col.back() + 1;
736 target_slot_idx += 2) {
738 const auto slot_off =
741 const auto desc =
"target_" +
std::to_string(target_logical_idx) +
"_second_slot";
743 this_targets_ptr, slot_off,
"this_" + desc);
745 that_targets_ptr, slot_off,
"that_" + desc);
755 slots_for_col.front(),
756 ir_reduce_one_entry);
757 auto increment_agg_val_idx_maybe =
758 [&init_agg_val_idx, &target_logical_idx,
this](
const int slot_count) {
761 init_agg_val_idx += slot_count;
764 if (target_logical_idx + 1 ==
targets_.size() &&
765 target_slot_idx + 1 >= slots_for_col.back()) {
768 const auto next_desc =
769 "target_" +
std::to_string(target_logical_idx + 1) +
"_first_slot";
771 increment_agg_val_idx_maybe(2);
772 const auto two_slot_off = ir_reduce_one_entry->addConstant<
ConstantInt>(
776 this_targets_ptr, two_slot_off,
"this_" + next_desc);
778 that_targets_ptr, two_slot_off,
"that_" + next_desc);
780 increment_agg_val_idx_maybe(1);
782 this_targets_ptr, slot_off,
"this_" + next_desc);
784 that_targets_ptr, slot_off,
"that_" + next_desc);
788 ir_reduce_one_entry->add<
Ret>(
795 const auto this_targets_ptr_arg = ir_reduce_one_entry->arg(0);
796 const auto that_targets_ptr_arg = ir_reduce_one_entry->arg(1);
797 Value* this_ptr1 = this_targets_ptr_arg;
798 Value* that_ptr1 = that_targets_ptr_arg;
800 size_t init_agg_val_idx = 0;
801 for (
size_t target_logical_idx = 0; target_logical_idx <
targets_.size();
802 ++target_logical_idx) {
803 const auto& target_info =
targets_[target_logical_idx];
804 Value* this_ptr2{
nullptr};
805 Value* that_ptr2{
nullptr};
806 if (target_info.is_agg &&
807 (target_info.agg_kind ==
kAVG ||
808 (target_info.agg_kind ==
kSAMPLE && target_info.sql_type.is_varlen()))) {
809 const auto desc =
"target_" +
std::to_string(target_logical_idx) +
"_second_slot";
810 const auto second_slot_rel_off =
813 this_ptr1, second_slot_rel_off,
"this_" + desc);
815 that_ptr1, second_slot_rel_off,
"that_" + desc);
826 ir_reduce_one_entry);
827 if (target_logical_idx + 1 ==
targets_.size()) {
831 init_agg_val_idx =
advance_slot(init_agg_val_idx, target_info,
false);
834 init_agg_val_idx =
advance_slot(init_agg_val_idx, target_info,
false);
838 const auto next_desc =
839 "target_" +
std::to_string(target_logical_idx + 1) +
"_first_slot";
840 auto next_slot_rel_off = ir_reduce_one_entry->addConstant<
ConstantInt>(
843 this_targets_ptr_arg, next_slot_rel_off, next_desc);
845 that_targets_ptr_arg, next_slot_rel_off, next_desc);
847 ir_reduce_one_entry->add<
Ret>(
858 const auto this_buff = ir_reduce_one_entry_idx->arg(0);
859 const auto that_buff = ir_reduce_one_entry_idx->arg(1);
860 const auto entry_idx = ir_reduce_one_entry_idx->arg(2);
861 const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
862 const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
863 const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
864 const auto row_bytes = ir_reduce_one_entry_idx->addConstant<
ConstantInt>(
866 const auto entry_idx_64 = ir_reduce_one_entry_idx->add<
Cast>(
868 const auto row_off_in_bytes = ir_reduce_one_entry_idx->add<
BinaryOperator>(
870 const auto this_row_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
871 this_buff, row_off_in_bytes,
"this_row_ptr");
872 const auto that_row_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
873 that_buff, row_off_in_bytes,
"that_row_ptr");
874 const auto reduce_rc = ir_reduce_one_entry_idx->add<
Call>(
876 std::vector<const Value*>{this_row_ptr,
880 serialized_varlen_buffer_arg},
882 ir_reduce_one_entry_idx->add<
Ret>(reduce_rc);
892 const auto this_buff = ir_reduce_one_entry_idx->arg(0);
893 const auto that_buff = ir_reduce_one_entry_idx->arg(1);
894 const auto that_entry_idx = ir_reduce_one_entry_idx->arg(2);
895 const auto that_entry_count = ir_reduce_one_entry_idx->arg(3);
896 const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
897 const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
898 const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
899 const auto row_bytes = ir_reduce_one_entry_idx->addConstant<
ConstantInt>(
901 const auto that_entry_idx_64 = ir_reduce_one_entry_idx->add<
Cast>(
903 const auto that_row_off_in_bytes =
907 "that_row_off_in_bytes");
908 const auto that_row_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
909 that_buff, that_row_off_in_bytes,
"that_row_ptr");
910 const auto that_is_empty =
912 std::vector<const Value*>{that_row_ptr},
919 const auto one_element =
921 const auto this_targets_ptr_i64_ptr = ir_reduce_one_entry_idx->add<
Alloca>(
923 const auto this_is_empty_ptr =
924 ir_reduce_one_entry_idx->add<
Alloca>(
Type::Int8, one_element,
"this_is_empty_out");
926 "get_group_value_reduction_rt",
928 std::vector<const Value*>{
937 this_targets_ptr_i64_ptr,
940 const auto this_targets_ptr_i64 = ir_reduce_one_entry_idx->add<
Load>(
941 this_targets_ptr_i64_ptr,
"this_targets_ptr_i64");
943 ir_reduce_one_entry_idx->add<
Load>(this_is_empty_ptr,
"this_is_empty");
944 this_is_empty = ir_reduce_one_entry_idx->add<
Cast>(
951 const auto this_targets_ptr = ir_reduce_one_entry_idx->add<
Cast>(
953 const auto key_byte_count = key_qw_count *
sizeof(int64_t);
954 const auto key_byte_count_lv =
956 const auto that_targets_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
957 that_row_ptr, key_byte_count_lv,
"that_targets_ptr");
958 const auto reduce_rc = ir_reduce_one_entry_idx->add<
Call>(
960 std::vector<const Value*>{this_targets_ptr,
964 serialized_varlen_buffer_arg},
966 ir_reduce_one_entry_idx->add<
Ret>(reduce_rc);
972 Function* ir_reduce_loop,
973 Function* ir_reduce_one_entry_idx,
977 Value* that_entry_count,
978 Value* this_qmd_handle,
979 Value* that_qmd_handle,
980 Value* serialized_varlen_buffer) {
983 const auto watchdog_sample_seed =
985 const auto watchdog_triggered =
988 std::vector<const Value*>{watchdog_sample_seed},
990 const auto watchdog_triggered_bool =
996 watchdog_triggered_bool,
999 const auto reduce_rc =
1000 for_loop->
add<
Call>(ir_reduce_one_entry_idx,
1001 std::vector<const Value*>{this_buff,
1007 serialized_varlen_buffer},
1010 auto reduce_rc_bool =
1022 const auto this_buff_arg = ir_reduce_loop->arg(0);
1023 const auto that_buff_arg = ir_reduce_loop->arg(1);
1024 const auto start_index_arg = ir_reduce_loop->arg(2);
1025 const auto end_index_arg = ir_reduce_loop->arg(3);
1026 const auto that_entry_count_arg = ir_reduce_loop->arg(4);
1027 const auto this_qmd_handle_arg = ir_reduce_loop->arg(5);
1028 const auto that_qmd_handle_arg = ir_reduce_loop->arg(6);
1029 const auto serialized_varlen_buffer_arg = ir_reduce_loop->arg(7);
1031 static_cast<For*
>(ir_reduce_loop->add<
For>(start_index_arg, end_index_arg,
""));
1038 that_entry_count_arg,
1039 this_qmd_handle_arg,
1040 that_qmd_handle_arg,
1041 serialized_varlen_buffer_arg);
1050 const size_t target_logical_idx,
1051 const size_t target_slot_idx,
1052 const size_t init_agg_val_idx,
1053 const size_t first_slot_idx_for_target,
1054 Function* ir_reduce_one_entry)
const {
1065 if (target_info.
is_agg &&
1076 ir_reduce_one_entry);
1079 this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1081 auto checked_rc_bool = ir_reduce_one_entry->add<
ICmp>(
1087 ir_reduce_one_entry->add<
ReturnEarly>(checked_rc_bool, checked_rc,
"");
1091 this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1093 CHECK(this_ptr2 && that_ptr2);
1094 size_t length_to_elems{0};
1097 length_to_elems = target_slot_idx == first_slot_idx_for_target ? 1 : 4;
1102 const auto serialized_varlen_buffer_arg = ir_reduce_one_entry->arg(4);
1104 "serialized_varlen_buffer_sample",
1106 std::vector<const Value*>{
1107 serialized_varlen_buffer_arg,
1113 ir_reduce_one_entry->addConstant<
ConstantInt>(length_to_elems,
1125 const size_t target_logical_idx,
1126 const size_t target_slot_idx,
1127 const int64_t init_val,
1128 const int8_t chosen_bytes,
1129 Function* ir_reduce_one_entry)
const {
1134 CHECK_EQ(static_cast<size_t>(chosen_bytes),
sizeof(int64_t));
1136 this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1144 CHECK_EQ(chosen_bytes, static_cast<int8_t>(
sizeof(int64_t)));
1146 this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1153 ir_reduce_one_entry);
1163 ir_reduce_one_entry);
1173 ir_reduce_one_entry);
1183 ir_reduce_one_entry);
1187 LOG(
FATAL) <<
"Invalid aggregate type";
1194 const size_t target_logical_idx,
1195 Function* ir_reduce_one_entry)
const {
1197 const auto old_set_handle =
emit_load_i64(this_ptr1, ir_reduce_one_entry);
1198 const auto new_set_handle =
emit_load_i64(that_ptr1, ir_reduce_one_entry);
1199 const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1200 const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1202 "count_distinct_set_union_jit_rt",
1204 std::vector<const Value*>{
1216 const size_t target_logical_idx,
1217 Function* ir_reduce_one_entry)
const {
1219 const auto old_set_handle =
emit_load_i64(this_ptr1, ir_reduce_one_entry);
1220 const auto new_set_handle =
emit_load_i64(that_ptr1, ir_reduce_one_entry);
1221 const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1222 const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1224 "approx_median_jit_rt",
1226 std::vector<const Value*>{
1237 const llvm::Function* ir_is_empty,
1238 const llvm::Function* ir_reduce_one_entry,
1239 const llvm::Function* ir_reduce_one_entry_idx,
1245 LOG(
IR) <<
"Reduction Loop:\n" 1249 LOG(
IR) <<
"Reduction One Entry Idx Func:\n" 1255 reduction_code.
module.release();
1261 auto cpu_compilation_context = std::make_shared<CpuCompilationContext>(std::move(ee));
1262 cpu_compilation_context->setFunctionPointer(reduction_code.
llvm_reduce_loop);
1268 return reduction_code;
1287 std::vector<std::string> target_init_vals_strings;
1290 std::back_inserter(target_init_vals_strings),
1292 const auto target_init_vals_key =
1294 std::vector<std::string> targets_strings;
1298 std::back_inserter(targets_strings),
1313 reduction_code.cgen_state.reset(
new CgenState({},
false));
1314 auto cgen_state = reduction_code.cgen_state.get();
1317 cgen_state->module_ = module.get();
1320 auto ir_reduce_one_entry =
1322 auto ir_reduce_one_entry_idx =
1324 auto ir_reduce_loop =
1326 std::unordered_map<const Function*, llvm::Function*> f;
1327 f.emplace(reduction_code.ir_is_empty.get(), ir_is_empty);
1328 f.emplace(reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry);
1329 f.emplace(reduction_code.ir_reduce_one_entry_idx.get(), ir_reduce_one_entry_idx);
1330 f.emplace(reduction_code.ir_reduce_loop.get(), ir_reduce_loop);
1331 translate_function(reduction_code.ir_is_empty.get(), ir_is_empty, reduction_code, f);
1333 reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code, f);
1335 ir_reduce_one_entry_idx,
1339 reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code, f);
1340 reduction_code.llvm_reduce_loop = ir_reduce_loop;
1341 reduction_code.module = std::move(module);
1342 return reduction_code;
GroupValueInfo get_group_value_reduction(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const size_t key_width, const QueryMemoryDescriptor &query_mem_desc, const int64_t *that_buff_i64, const size_t that_entry_idx, const size_t that_entry_count, const uint32_t row_size_quad)
void emit_aggregate_one_nullable_value(const std::string &agg_kind, Value *val_ptr, Value *other_ptr, const int64_t init_val, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
std::unique_ptr< CgenState > cgen_state
Value * add(Args &&... args)
void reduceOneEntryBaseline(const ReductionCode &reduction_code) const
bool is_aggregate_query(const QueryDescriptionType hash_type)
void isEmpty(const ReductionCode &reduction_code) const
void reduceOneEntryNoCollisionsIdx(const ReductionCode &reduction_code) const
void count_distinct_set_union(const int64_t new_set_handle, const int64_t old_set_handle, const CountDistinctDescriptor &new_count_distinct_desc, const CountDistinctDescriptor &old_count_distinct_desc)
bool didOutputColumnar() const
void count_distinct_set_union_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
std::unique_ptr< llvm::Module > runtime_module_shallow_copy(CgenState *cgen_state)
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function *> &live_funcs, const CompilationOptions &co)
void reduceOneApproxMedianSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
void serialized_varlen_buffer_sample(const void *serialized_varlen_buffer_handle, int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val, const int64_t length_to_elems)
void reduceOneSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const size_t init_agg_val_idx, const size_t first_slot_idx_for_target, Function *ir_reduce_one_entry) const
std::shared_ptr< CompilationContext > compilation_context
void varlen_buffer_sample(int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val)
std::unique_ptr< Function > ir_reduce_loop
Value * emit_read_int_from_buff(Value *ptr, const int8_t compact_sz, Function *function)
void mark_function_always_inline(llvm::Function *func)
void get_group_value_reduction_rt(int8_t *groups_buffer, const int8_t *key, const uint32_t key_count, const void *this_qmd_handle, const int8_t *that_buff, const uint32_t that_entry_idx, const uint32_t that_entry_count, const uint32_t row_size_bytes, int64_t **buff_out, uint8_t *empty)
const size_t INTERP_THRESHOLD
size_t getCountDistinctDescriptorsSize() const
llvm::Function * llvm_reduce_loop
void reduceOneEntryTargetsNoCollisions(Function *ir_reduce_one_entry, Value *this_targets_start_ptr, Value *that_targets_start_ptr) const
std::vector< std::string > CodeCacheKey
std::unique_ptr< llvm::Module > g_rt_module
size_t get_slot_off_quad(const QueryMemoryDescriptor &query_mem_desc)
std::unique_ptr< Function > ir_reduce_one_entry
bool g_enable_dynamic_watchdog
const std::vector< int64_t > target_init_vals_
bool takes_float_argument(const TargetInfo &target_info)
const Value * emit_checked_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
std::unique_ptr< Function > setup_reduce_one_entry_idx(ReductionCode *reduction_code)
int32_t(*)(int8_t *this_buff, const int8_t *that_buff, const int32_t start_entry_index, const int32_t end_entry_index, const int32_t that_entry_count, const void *this_qmd, const void *that_qmd, const void *serialized_varlen_buffer) FuncPtr
const QueryMemoryDescriptor query_mem_desc_
std::unique_ptr< Function > ir_is_empty
virtual ReductionCode codegen() const
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
void emit_aggregate_one_value(const std::string &agg_kind, Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
const Executor * getExecutor() const
Value * emit_load_i32(Value *ptr, Function *function)
const SQLTypeInfo get_compact_type(const TargetInfo &target)
HOST DEVICE bool get_notnull() const
int8_t get_width_for_slot(const size_t target_slot_idx, const bool float_argument_input, const QueryMemoryDescriptor &query_mem_desc)
ResultSetReductionJIT(const QueryMemoryDescriptor &query_mem_desc, const std::vector< TargetInfo > &targets, const std::vector< int64_t > &target_init_vals)
llvm::LLVMContext & context_
size_t get_byteoff_of_slot(const size_t slot_idx, const QueryMemoryDescriptor &query_mem_desc)
size_t advance_slot(const size_t j, const TargetInfo &target_info, const bool separate_varlen_storage)
bool blocksShareMemory() const
ReductionCode finalizeReductionCode(ReductionCode reduction_code, const llvm::Function *ir_is_empty, const llvm::Function *ir_reduce_one_entry, const llvm::Function *ir_reduce_one_entry_idx, const CodeCacheKey &key) const
void generate_loop_body(For *for_loop, Function *ir_reduce_loop, Function *ir_reduce_one_entry_idx, Value *this_buff, Value *that_buff, Value *start_index, Value *that_entry_count, Value *this_qmd_handle, Value *that_qmd_handle, Value *serialized_varlen_buffer)
void emit_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
void reduceOneEntryNoCollisions(const ReductionCode &reduction_code) const
void translate_function(const Function *function, llvm::Function *llvm_function, const ReductionCode &reduction_code, const std::unordered_map< const Function *, llvm::Function *> &f)
DEVICE void mergeTDigest(TDigest &t_digest)
uint8_t check_watchdog_rt(const size_t sample_seed)
static std::mutex s_reduction_mutex
bool is_distinct_target(const TargetInfo &target_info)
std::string target_info_key(const TargetInfo &target_info)
std::unique_ptr< Function > ir_reduce_one_entry_idx
ReductionCode setup_functions_ir(const QueryDescriptionType hash_type)
size_t targetGroupbyIndicesSize() const
std::unique_ptr< Function > setup_is_empty_entry(ReductionCode *reduction_code)
static void addCodeToCache(const CodeCacheKey &, std::shared_ptr< CompilationContext >, llvm::Module *, CodeCache &)
llvm::Type * llvm_type(const Type type, llvm::LLVMContext &ctx)
int32_t getTargetIdxForKey() const
void reduceOneCountDistinctSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
static CodeCache s_code_cache
size_t getGroupbyColCount() const
std::string serialize_llvm_object(const T *llvm_obj)
std::string get_type_name() const
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
int64_t getTargetGroupbyIndex(const size_t target_idx) const
bool hasKeylessHash() const
void reduceOneEntryBaselineIdx(const ReductionCode &reduction_code) const
void approx_median_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
std::unique_ptr< Function > create_function(const std::string name, const std::vector< Function::NamedArg > &arg_types, const Type ret_type, const bool always_inline)
std::string reductionKey() const
std::unique_ptr< Function > setup_reduce_one_entry(ReductionCode *reduction_code, const QueryDescriptionType hash_type)
Value * emit_load_i64(Value *ptr, Function *function)
SQLTypeInfo get_elem_type() const
const ColSlotContext & getColSlotContext() const
std::unique_ptr< llvm::Module > module
const int32_t WATCHDOG_ERROR
const Value * iter() const
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
Value * emit_load(Value *ptr, Type ptr_type, Function *function)
std::string cacheKey() const
value_t * get(const key_t &key)
llvm::Function * create_llvm_function(const Function *function, CgenState *cgen_state)
void reduceLoop(const ReductionCode &reduction_code) const
void emit_aggregate_one_count(Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, Function *ir_reduce_one_entry)
QueryDescriptionType getQueryDescriptionType() const
virtual ReductionCode codegen() const
std::unique_ptr< Function > setup_reduce_loop(ReductionCode *reduction_code)
size_t getEntryCount() const
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
const std::string & label() const
size_t getEffectiveKeyWidth() const
void reduceOneAggregateSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const int64_t init_val, const int8_t chosen_bytes, Function *ir_reduce_one_entry) const
const std::vector< TargetInfo > targets_