31 #include <llvm/Bitcode/BitcodeReader.h>
32 #include <llvm/IR/Function.h>
33 #include <llvm/IR/IRBuilder.h>
34 #include <llvm/IR/Verifier.h>
35 #include <llvm/Support/SourceMgr.h>
36 #include <llvm/Support/raw_os_ostream.h>
49 return function->add<
Load>(
51 ptr->
label() +
"_loaded");
75 LOG(
FATAL) <<
"Invalid byte width: " << compact_sz;
86 const size_t chosen_bytes,
88 Function* ir_reduce_one_entry) {
90 const auto dest_name = agg_kind +
"_dest";
91 if (sql_type.is_fp()) {
92 if (chosen_bytes ==
sizeof(
float)) {
93 const auto agg = ir_reduce_one_entry->add<
Cast>(
96 ir_reduce_one_entry->add<
Call>(
97 "agg_" + agg_kind +
"_float", std::vector<const Value*>{agg, val},
"");
99 CHECK_EQ(chosen_bytes,
sizeof(
double));
100 const auto agg = ir_reduce_one_entry->add<
Cast>(
103 ir_reduce_one_entry->add<
Call>(
104 "agg_" + agg_kind +
"_double", std::vector<const Value*>{agg, val},
"");
107 if (chosen_bytes ==
sizeof(int32_t)) {
108 const auto agg = ir_reduce_one_entry->add<
Cast>(
110 const auto val =
emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
111 ir_reduce_one_entry->add<
Call>(
112 "agg_" + agg_kind +
"_int32", std::vector<const Value*>{agg, val},
"");
114 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
115 const auto agg = ir_reduce_one_entry->add<
Cast>(
117 const auto val =
emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
118 ir_reduce_one_entry->add<
Call>(
119 "agg_" + agg_kind, std::vector<const Value*>{agg, val},
"");
128 const int64_t init_val,
129 const size_t chosen_bytes,
131 Function* ir_reduce_one_entry) {
133 const auto dest_name = agg_kind +
"_dest";
136 if (sql_type.is_fp()) {
137 if (chosen_bytes ==
sizeof(
float)) {
138 const auto agg = ir_reduce_one_entry->add<
Cast>(
141 const auto init_val_lv = ir_reduce_one_entry->addConstant<
ConstantFP>(
142 *
reinterpret_cast<const float*
>(may_alias_ptr(&init_val)),
Type::Float);
143 std::vector<const Value*>
args{agg, val, init_val_lv};
144 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_float_skip_val",
args,
"");
146 CHECK_EQ(chosen_bytes,
sizeof(
double));
147 const auto agg = ir_reduce_one_entry->add<
Cast>(
150 const auto init_val_lv = ir_reduce_one_entry->addConstant<
ConstantFP>(
151 *
reinterpret_cast<const double*
>(may_alias_ptr(&init_val)),
Type::Double);
152 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_double_skip_val",
153 std::vector<const Value*>{agg, val, init_val_lv},
157 if (chosen_bytes ==
sizeof(int32_t)) {
158 const auto agg = ir_reduce_one_entry->add<
Cast>(
160 const auto val =
emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
161 const auto init_val_lv =
163 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_int32_skip_val",
164 std::vector<const Value*>{agg, val, init_val_lv},
167 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
168 const auto agg = ir_reduce_one_entry->add<
Cast>(
170 const auto val =
emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
171 const auto init_val_lv =
173 ir_reduce_one_entry->add<
Call>(
"agg_" + agg_kind +
"_skip_val",
174 std::vector<const Value*>{agg, val, init_val_lv},
180 agg_kind, val_ptr, other_ptr, chosen_bytes, agg_info, ir_reduce_one_entry);
187 const size_t chosen_bytes,
188 Function* ir_reduce_one_entry) {
189 const auto dest_name =
"count_dest";
190 if (chosen_bytes ==
sizeof(int32_t)) {
191 const auto agg = ir_reduce_one_entry->add<
Cast>(
193 const auto val =
emit_load(other_ptr, Type::Int32Ptr, ir_reduce_one_entry);
194 ir_reduce_one_entry->add<
Call>(
195 "agg_sum_int32", std::vector<const Value*>{agg, val},
"");
197 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
198 const auto agg = ir_reduce_one_entry->add<
Cast>(
200 const auto val =
emit_load(other_ptr, Type::Int64Ptr, ir_reduce_one_entry);
201 ir_reduce_one_entry->add<
Call>(
"agg_sum", std::vector<const Value*>{agg, val},
"");
210 const int64_t init_val,
211 const size_t chosen_bytes,
212 Function* ir_reduce_one_entry) {
213 const auto func_name =
"write_projection_int" +
std::to_string(chosen_bytes * 8);
214 if (chosen_bytes ==
sizeof(int32_t)) {
215 const auto proj_val =
emit_load_i32(other_pi8, ir_reduce_one_entry);
216 ir_reduce_one_entry->add<
Call>(
218 std::vector<const Value*>{
224 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
225 const auto proj_val =
emit_load_i64(other_pi8, ir_reduce_one_entry);
226 ir_reduce_one_entry->add<
Call>(
228 std::vector<const Value*>{
241 const int64_t init_val,
242 const size_t chosen_bytes,
243 Function* ir_reduce_one_entry) {
244 if (chosen_bytes ==
sizeof(int32_t)) {
245 const auto func_name =
"checked_single_agg_id_int32";
246 const auto proj_val =
emit_load_i32(other_pi8, ir_reduce_one_entry);
247 const auto slot_pi32 = ir_reduce_one_entry->add<
Cast>(
249 return ir_reduce_one_entry->add<
Call>(
252 std::vector<const Value*>{
258 const auto func_name =
"checked_single_agg_id";
259 CHECK_EQ(chosen_bytes,
sizeof(int64_t));
260 const auto proj_val =
emit_load_i64(other_pi8, ir_reduce_one_entry);
261 const auto slot_pi64 = ir_reduce_one_entry->add<
Cast>(
264 return ir_reduce_one_entry->add<
Call>(
267 std::vector<const Value*>{
276 const std::string
name,
277 const std::vector<Function::NamedArg>& arg_types,
279 const bool always_inline) {
280 return std::make_unique<Function>(
name, arg_types, ret_type, always_inline);
293 std::string this_ptr_name;
294 std::string that_ptr_name;
297 this_ptr_name =
"this_targets_ptr";
298 that_ptr_name =
"that_targets_ptr";
303 this_ptr_name =
"this_row_ptr";
304 that_ptr_name =
"that_row_ptr";
308 LOG(
FATAL) <<
"Unexpected query description type";
326 {
"that_entry_idx", Type::Int32},
327 {
"that_entry_count", Type::Int32},
341 {
"start_index", Type::Int32},
342 {
"end_index", Type::Int32},
343 {
"that_entry_count", Type::Int32},
354 std::vector<llvm::Type*> parameter_types;
355 const auto& arg_types =
function->arg_types();
356 for (
const auto& named_arg : arg_types) {
358 parameter_types.push_back(
llvm_type(named_arg.type, ctx));
360 const auto func_type = llvm::FunctionType::get(
361 llvm_type(function->ret_type(), ctx), parameter_types,
false);
362 const auto linkage =
function->always_inline() ? llvm::Function::PrivateLinkage
363 : llvm::Function::ExternalLinkage;
365 llvm::Function::Create(func_type, linkage, function->name(), cgen_state->
module_);
366 const auto arg_it = func->arg_begin();
367 for (
size_t i = 0; i < arg_types.size(); ++i) {
368 const auto arg = &*(arg_it + i);
369 arg->setName(arg_types[i].
name);
371 if (function->always_inline()) {
385 return reduction_code;
397 const int8_t* that_ptr1,
398 const int8_t* that_ptr2,
399 const int64_t init_val) {
400 const auto rhs_proj_col = *
reinterpret_cast<const int64_t*
>(that_ptr1);
401 if (rhs_proj_col != init_val) {
402 *
reinterpret_cast<int64_t*
>(this_ptr1) = rhs_proj_col;
404 CHECK(this_ptr2 && that_ptr2);
405 *
reinterpret_cast<int64_t*
>(this_ptr2) = *reinterpret_cast<const int64_t*>(that_ptr2);
411 const void* serialized_varlen_buffer_handle,
414 const int8_t* that_ptr1,
415 const int8_t* that_ptr2,
416 const int64_t init_val,
417 const int64_t length_to_elems) {
418 if (!serialized_varlen_buffer_handle) {
422 const auto& serialized_varlen_buffer =
423 *
reinterpret_cast<const std::vector<std::string>*
>(serialized_varlen_buffer_handle);
424 if (!serialized_varlen_buffer.empty()) {
425 const auto rhs_proj_col = *
reinterpret_cast<const int64_t*
>(that_ptr1);
426 CHECK_LT(static_cast<size_t>(rhs_proj_col), serialized_varlen_buffer.size());
427 const auto& varlen_bytes_str = serialized_varlen_buffer[rhs_proj_col];
428 const auto str_ptr =
reinterpret_cast<const int8_t*
>(varlen_bytes_str.c_str());
429 *
reinterpret_cast<int64_t*
>(this_ptr1) = reinterpret_cast<const int64_t>(str_ptr);
430 *
reinterpret_cast<int64_t*
>(this_ptr2) =
431 static_cast<int64_t>(varlen_bytes_str.size() / length_to_elems);
441 const int64_t new_set_handle,
442 const int64_t old_set_handle,
443 const void* that_qmd_handle,
444 const void* this_qmd_handle,
445 const int64_t target_logical_idx) {
448 const auto& new_count_distinct_desc =
450 const auto& old_count_distinct_desc =
451 this_qmd->getCountDistinctDescriptor(target_logical_idx);
453 CHECK(old_count_distinct_desc.impl_type_ == new_count_distinct_desc.impl_type_);
455 new_set_handle, old_set_handle, new_count_distinct_desc, old_count_distinct_desc);
459 const int64_t old_set_handle,
460 const void* that_qmd_handle,
461 const void* this_qmd_handle,
462 const int64_t target_logical_idx) {
464 if (incoming->centroids().capacity()) {
467 accumulator->mergeTDigest(*incoming);
472 const int64_t old_set_handle,
473 const void* that_qmd_handle,
474 const void* this_qmd_handle,
475 const int64_t target_logical_idx) {
476 auto* accumulator =
reinterpret_cast<AggMode*
>(old_set_handle);
477 auto* incoming =
reinterpret_cast<AggMode*
>(new_set_handle);
478 accumulator->
reduce(std::move(*incoming));
482 int8_t* groups_buffer,
484 const uint32_t key_count,
485 const void* this_qmd_handle,
486 const int8_t* that_buff,
487 const uint32_t that_entry_idx,
488 const uint32_t that_entry_count,
489 const uint32_t row_size_bytes,
495 this_qmd.getEntryCount(),
496 reinterpret_cast<const int64_t*
>(key),
498 this_qmd.getEffectiveKeyWidth(),
500 reinterpret_cast<const int64_t*
>(that_buff),
503 row_size_bytes >> 3);
504 *buff_out = gvi.first;
525 const std::vector<TargetInfo>& targets,
526 const std::vector<int64_t>& target_init_vals,
527 const size_t executor_id)
528 : executor_id_(executor_id)
529 , query_mem_desc_(query_mem_desc)
531 , target_init_vals_(target_init_vals) {}
584 LOG(
FATAL) <<
"Unexpected query description type";
591 return reduction_code;
595 std::lock_guard<std::mutex> compilation_lock(executor->compilation_mutex_);
596 const auto compilation_context =
598 if (compilation_context) {
599 reduction_code.func_ptr =
601 return reduction_code;
603 auto cgen_state_ = std::unique_ptr<CgenState>(
new CgenState({},
false, executor.get()));
604 auto cgen_state = reduction_code.cgen_state = cgen_state_.get();
605 cgen_state->set_module_shallow_copy(executor->get_rt_module());
606 reduction_code.module = cgen_state->module_;
610 auto ir_reduce_one_entry =
612 auto ir_reduce_one_entry_idx =
614 auto ir_reduce_loop =
616 std::unordered_map<const Function*, llvm::Function*>
f;
617 f.emplace(reduction_code.ir_is_empty.get(), ir_is_empty);
618 f.emplace(reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry);
619 f.emplace(reduction_code.ir_reduce_one_entry_idx.get(), ir_reduce_one_entry_idx);
620 f.emplace(reduction_code.ir_reduce_loop.get(), ir_reduce_loop);
623 reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code,
f);
625 ir_reduce_one_entry_idx,
629 reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code,
f);
630 reduction_code.llvm_reduce_loop = ir_reduce_loop;
632 reduction_code.cgen_state =
nullptr;
634 reduction_code, ir_is_empty, ir_reduce_one_entry, ir_reduce_one_entry_idx, key);
635 return reduction_code;
639 auto ir_is_empty = reduction_code.
ir_is_empty.get();
643 Value* empty_key_val{
nullptr};
644 const auto keys_ptr = ir_is_empty->arg(0);
656 "is_empty_slot_ptr");
657 const auto compact_sz =
660 empty_key_val = ir_is_empty->addConstant<
ConstantInt>(
682 ir_is_empty->add<
Ret>(ret);
688 const auto this_row_ptr = ir_reduce_one_entry->arg(0);
689 const auto that_row_ptr = ir_reduce_one_entry->arg(1);
690 const auto that_is_empty =
692 std::vector<const Value*>{that_row_ptr},
699 ir_reduce_one_entry->add<
MemCpy>(
706 const auto key_bytes_lv =
708 const auto this_targets_start_ptr = ir_reduce_one_entry->add<
GetElementPtr>(
709 this_row_ptr, key_bytes_lv,
"this_targets_start");
710 const auto that_targets_start_ptr = ir_reduce_one_entry->add<
GetElementPtr>(
711 that_row_ptr, key_bytes_lv,
"that_targets_start");
714 ir_reduce_one_entry, this_targets_start_ptr, that_targets_start_ptr);
718 Function* ir_reduce_one_entry,
719 Value* this_targets_start_ptr,
720 Value* that_targets_start_ptr)
const {
722 Value* this_targets_ptr = this_targets_start_ptr;
723 Value* that_targets_ptr = that_targets_start_ptr;
724 size_t init_agg_val_idx = 0;
725 for (
size_t target_logical_idx = 0; target_logical_idx <
targets_.size();
726 ++target_logical_idx) {
727 const auto& target_info =
targets_[target_logical_idx];
728 const auto& slots_for_col = col_slot_context.getSlotsForCol(target_logical_idx);
729 Value* this_ptr2{
nullptr};
730 Value* that_ptr2{
nullptr};
732 bool two_slot_target{
false};
733 if (target_info.is_agg &&
734 (target_info.agg_kind ==
kAVG ||
735 (target_info.agg_kind ==
kSAMPLE && target_info.sql_type.is_varlen()))) {
739 two_slot_target =
true;
742 for (
size_t target_slot_idx = slots_for_col.front();
743 target_slot_idx < slots_for_col.back() + 1;
744 target_slot_idx += 2) {
746 const auto slot_off =
749 const auto desc =
"target_" +
std::to_string(target_logical_idx) +
"_second_slot";
751 this_targets_ptr, slot_off,
"this_" + desc);
753 that_targets_ptr, slot_off,
"that_" + desc);
763 slots_for_col.front(),
764 ir_reduce_one_entry);
765 auto increment_agg_val_idx_maybe =
766 [&init_agg_val_idx, &target_logical_idx,
this](
const int slot_count) {
769 init_agg_val_idx += slot_count;
772 if (target_logical_idx + 1 ==
targets_.size() &&
773 target_slot_idx + 1 >= slots_for_col.back()) {
776 const auto next_desc =
777 "target_" +
std::to_string(target_logical_idx + 1) +
"_first_slot";
779 increment_agg_val_idx_maybe(2);
780 const auto two_slot_off = ir_reduce_one_entry->addConstant<
ConstantInt>(
784 this_targets_ptr, two_slot_off,
"this_" + next_desc);
786 that_targets_ptr, two_slot_off,
"that_" + next_desc);
788 increment_agg_val_idx_maybe(1);
790 this_targets_ptr, slot_off,
"this_" + next_desc);
792 that_targets_ptr, slot_off,
"that_" + next_desc);
796 ir_reduce_one_entry->add<
Ret>(
803 const auto this_targets_ptr_arg = ir_reduce_one_entry->arg(0);
804 const auto that_targets_ptr_arg = ir_reduce_one_entry->arg(1);
805 Value* this_ptr1 = this_targets_ptr_arg;
806 Value* that_ptr1 = that_targets_ptr_arg;
808 size_t init_agg_val_idx = 0;
809 for (
size_t target_logical_idx = 0; target_logical_idx <
targets_.size();
810 ++target_logical_idx) {
811 const auto& target_info =
targets_[target_logical_idx];
812 Value* this_ptr2{
nullptr};
813 Value* that_ptr2{
nullptr};
814 if (target_info.is_agg &&
815 (target_info.agg_kind ==
kAVG ||
816 (target_info.agg_kind ==
kSAMPLE && target_info.sql_type.is_varlen()))) {
817 const auto desc =
"target_" +
std::to_string(target_logical_idx) +
"_second_slot";
818 const auto second_slot_rel_off =
821 this_ptr1, second_slot_rel_off,
"this_" + desc);
823 that_ptr1, second_slot_rel_off,
"that_" + desc);
834 ir_reduce_one_entry);
835 if (target_logical_idx + 1 ==
targets_.size()) {
839 init_agg_val_idx =
advance_slot(init_agg_val_idx, target_info,
false);
842 init_agg_val_idx =
advance_slot(init_agg_val_idx, target_info,
false);
846 const auto next_desc =
847 "target_" +
std::to_string(target_logical_idx + 1) +
"_first_slot";
848 auto next_slot_rel_off = ir_reduce_one_entry->addConstant<
ConstantInt>(
851 this_targets_ptr_arg, next_slot_rel_off, next_desc);
853 that_targets_ptr_arg, next_slot_rel_off, next_desc);
855 ir_reduce_one_entry->add<
Ret>(
866 const auto this_buff = ir_reduce_one_entry_idx->arg(0);
867 const auto that_buff = ir_reduce_one_entry_idx->arg(1);
868 const auto entry_idx = ir_reduce_one_entry_idx->arg(2);
869 const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
870 const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
871 const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
872 const auto row_bytes = ir_reduce_one_entry_idx->addConstant<
ConstantInt>(
874 const auto entry_idx_64 = ir_reduce_one_entry_idx->add<
Cast>(
876 const auto row_off_in_bytes = ir_reduce_one_entry_idx->add<
BinaryOperator>(
878 const auto this_row_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
879 this_buff, row_off_in_bytes,
"this_row_ptr");
880 const auto that_row_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
881 that_buff, row_off_in_bytes,
"that_row_ptr");
882 const auto reduce_rc = ir_reduce_one_entry_idx->add<
Call>(
884 std::vector<const Value*>{this_row_ptr,
888 serialized_varlen_buffer_arg},
890 ir_reduce_one_entry_idx->add<
Ret>(reduce_rc);
900 const auto this_buff = ir_reduce_one_entry_idx->arg(0);
901 const auto that_buff = ir_reduce_one_entry_idx->arg(1);
902 const auto that_entry_idx = ir_reduce_one_entry_idx->arg(2);
903 const auto that_entry_count = ir_reduce_one_entry_idx->arg(3);
904 const auto this_qmd_handle = ir_reduce_one_entry_idx->arg(4);
905 const auto that_qmd_handle = ir_reduce_one_entry_idx->arg(5);
906 const auto serialized_varlen_buffer_arg = ir_reduce_one_entry_idx->arg(6);
907 const auto row_bytes = ir_reduce_one_entry_idx->addConstant<
ConstantInt>(
909 const auto that_entry_idx_64 = ir_reduce_one_entry_idx->add<
Cast>(
911 const auto that_row_off_in_bytes =
915 "that_row_off_in_bytes");
916 const auto that_row_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
917 that_buff, that_row_off_in_bytes,
"that_row_ptr");
918 const auto that_is_empty =
920 std::vector<const Value*>{that_row_ptr},
927 const auto one_element =
929 const auto this_targets_ptr_i64_ptr = ir_reduce_one_entry_idx->add<
Alloca>(
931 const auto this_is_empty_ptr =
932 ir_reduce_one_entry_idx->add<
Alloca>(
Type::Int8, one_element,
"this_is_empty_out");
934 "get_group_value_reduction_rt",
936 std::vector<const Value*>{
945 this_targets_ptr_i64_ptr,
948 const auto this_targets_ptr_i64 = ir_reduce_one_entry_idx->add<
Load>(
949 this_targets_ptr_i64_ptr,
"this_targets_ptr_i64");
951 ir_reduce_one_entry_idx->add<
Load>(this_is_empty_ptr,
"this_is_empty");
952 this_is_empty = ir_reduce_one_entry_idx->add<
Cast>(
959 const auto this_targets_ptr = ir_reduce_one_entry_idx->add<
Cast>(
961 const auto key_byte_count = key_qw_count *
sizeof(int64_t);
962 const auto key_byte_count_lv =
964 const auto that_targets_ptr = ir_reduce_one_entry_idx->add<
GetElementPtr>(
965 that_row_ptr, key_byte_count_lv,
"that_targets_ptr");
966 const auto reduce_rc = ir_reduce_one_entry_idx->add<
Call>(
968 std::vector<const Value*>{this_targets_ptr,
972 serialized_varlen_buffer_arg},
974 ir_reduce_one_entry_idx->add<
Ret>(reduce_rc);
980 Function* ir_reduce_loop,
981 Function* ir_reduce_one_entry_idx,
985 Value* that_entry_count,
986 Value* this_qmd_handle,
987 Value* that_qmd_handle,
988 Value* serialized_varlen_buffer) {
991 const auto sample_seed =
994 const auto checker_rt_name =
998 checker_rt_name,
Type::Int8, std::vector<const Value*>{sample_seed},
"");
999 const auto interrupt_triggered_bool =
1005 interrupt_triggered_bool,
1009 const auto reduce_rc =
1010 for_loop->
add<
Call>(ir_reduce_one_entry_idx,
1011 std::vector<const Value*>{this_buff,
1017 serialized_varlen_buffer},
1020 auto reduce_rc_bool =
1032 const auto this_buff_arg = ir_reduce_loop->arg(0);
1033 const auto that_buff_arg = ir_reduce_loop->arg(1);
1034 const auto start_index_arg = ir_reduce_loop->arg(2);
1035 const auto end_index_arg = ir_reduce_loop->arg(3);
1036 const auto that_entry_count_arg = ir_reduce_loop->arg(4);
1037 const auto this_qmd_handle_arg = ir_reduce_loop->arg(5);
1038 const auto that_qmd_handle_arg = ir_reduce_loop->arg(6);
1039 const auto serialized_varlen_buffer_arg = ir_reduce_loop->arg(7);
1041 static_cast<For*
>(ir_reduce_loop->add<
For>(start_index_arg, end_index_arg,
""));
1048 that_entry_count_arg,
1049 this_qmd_handle_arg,
1050 that_qmd_handle_arg,
1051 serialized_varlen_buffer_arg);
1060 const size_t target_logical_idx,
1061 const size_t target_slot_idx,
1062 const size_t init_agg_val_idx,
1063 const size_t first_slot_idx_for_target,
1064 Function* ir_reduce_one_entry)
const {
1075 if (target_info.
is_agg &&
1086 ir_reduce_one_entry);
1089 this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1091 auto checked_rc_bool = ir_reduce_one_entry->add<
ICmp>(
1097 ir_reduce_one_entry->add<
ReturnEarly>(checked_rc_bool, checked_rc,
"");
1101 this_ptr1, that_ptr1, init_val, chosen_bytes, ir_reduce_one_entry);
1103 CHECK(this_ptr2 && that_ptr2);
1104 size_t length_to_elems{0};
1107 length_to_elems = target_slot_idx == first_slot_idx_for_target ? 1 : 4;
1112 const auto serialized_varlen_buffer_arg = ir_reduce_one_entry->arg(4);
1114 "serialized_varlen_buffer_sample",
1116 std::vector<const Value*>{
1117 serialized_varlen_buffer_arg,
1123 ir_reduce_one_entry->addConstant<
ConstantInt>(length_to_elems,
1135 const size_t target_logical_idx,
1136 const size_t target_slot_idx,
1137 const int64_t init_val,
1138 const int8_t chosen_bytes,
1139 Function* ir_reduce_one_entry)
const {
1140 auto agg_kind = target_info.
agg_kind;
1146 CHECK_EQ(static_cast<size_t>(chosen_bytes),
sizeof(int64_t));
1148 this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1155 CHECK_EQ(
sizeof(int64_t), static_cast<size_t>(chosen_bytes));
1157 this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1160 reduceOneModeSlot(this_ptr1, that_ptr1, target_logical_idx, ir_reduce_one_entry);
1167 ir_reduce_one_entry);
1182 ir_reduce_one_entry);
1185 UNREACHABLE() <<
"Invalid aggregate type: " << agg_kind;
1192 const size_t target_logical_idx,
1193 Function* ir_reduce_one_entry)
const {
1195 const auto old_set_handle =
emit_load_i64(this_ptr1, ir_reduce_one_entry);
1196 const auto new_set_handle =
emit_load_i64(that_ptr1, ir_reduce_one_entry);
1197 const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1198 const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1200 "count_distinct_set_union_jit_rt",
1202 std::vector<const Value*>{
1214 const size_t target_logical_idx,
1215 Function* ir_reduce_one_entry)
const {
1217 const auto old_set_handle =
emit_load_i64(this_ptr1, ir_reduce_one_entry);
1218 const auto new_set_handle =
emit_load_i64(that_ptr1, ir_reduce_one_entry);
1219 const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1220 const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1222 "approx_quantile_jit_rt",
1224 std::vector<const Value*>{
1235 const size_t target_logical_idx,
1236 Function* ir_reduce_one_entry)
const {
1238 const auto old_set_handle =
emit_load_i64(this_ptr1, ir_reduce_one_entry);
1239 const auto new_set_handle =
emit_load_i64(that_ptr1, ir_reduce_one_entry);
1240 const auto this_qmd_arg = ir_reduce_one_entry->arg(2);
1241 const auto that_qmd_arg = ir_reduce_one_entry->arg(3);
1245 std::vector<const Value*>{
1256 const llvm::Function* ir_is_empty,
1257 const llvm::Function* ir_reduce_one_entry,
1258 const llvm::Function* ir_reduce_one_entry_idx,
1262 VLOG(3) <<
"Reduction Loop:\n"
1266 VLOG(3) <<
"Reduction One Entry Idx Func:\n"
1269 LOG(
IR) <<
"Reduction Loop:\n"
1273 LOG(
IR) <<
"Reduction One Entry Idx Func:\n"
1280 auto cpu_compilation_context = std::make_shared<CpuCompilationContext>(std::move(ee));
1281 cpu_compilation_context->setFunctionPointer(reduction_code.
llvm_reduce_loop);
1287 std::move(cpu_compilation_context));
1306 std::vector<std::string> target_init_vals_strings;
1309 std::back_inserter(target_init_vals_strings),
1311 const auto target_init_vals_key =
1313 std::vector<std::string> targets_strings;
1317 std::back_inserter(targets_strings),
1333 auto cgen_state_ = std::unique_ptr<CgenState>(
new CgenState({},
false, executor.get()));
1334 auto cgen_state = reduction_code.
cgen_state = cgen_state_.get();
1337 reduction_code.
module = cgen_state->module_;
1341 auto ir_reduce_one_entry =
1343 auto ir_reduce_one_entry_idx =
1345 auto ir_reduce_loop =
1347 std::unordered_map<const Function*, llvm::Function*>
f;
1348 f.emplace(reduction_code.
ir_is_empty.get(), ir_is_empty);
1354 reduction_code.ir_reduce_one_entry.get(), ir_reduce_one_entry, reduction_code,
f);
1356 ir_reduce_one_entry_idx,
1360 reduction_code.ir_reduce_loop.get(), ir_reduce_loop, reduction_code,
f);
1361 reduction_code.llvm_reduce_loop = ir_reduce_loop;
1362 reduction_code.cgen_state =
nullptr;
1363 return reduction_code;
GroupValueInfo get_group_value_reduction(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const size_t key_width, const QueryMemoryDescriptor &query_mem_desc, const int64_t *that_buff_i64, const size_t that_entry_idx, const size_t that_entry_count, const uint32_t row_size_quad)
void reduceOneSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const size_t init_agg_val_idx, const size_t first_slot_idx_for_target, Function *ir_reduce_one_entry) const
void reduce(AggMode &&rhs)
bool is_aggregate_query(const QueryDescriptionType hash_type)
size_t getEntryCount() const
void count_distinct_set_union(const int64_t new_set_handle, const int64_t old_set_handle, const CountDistinctDescriptor &new_count_distinct_desc, const CountDistinctDescriptor &old_count_distinct_desc)
__device__ bool dynamic_watchdog()
const std::string & label() const
RUNTIME_EXPORT uint8_t check_watchdog_rt(const size_t sample_seed)
void reduceOneEntryNoCollisions(const ReductionCode &reduction_code) const
int64_t getTargetGroupbyIndex(const size_t target_idx) const
void varlen_buffer_sample(int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val)
std::unique_ptr< Function > ir_reduce_loop
Value * emit_read_int_from_buff(Value *ptr, const int8_t compact_sz, Function *function)
void reduceOneEntryBaselineIdx(const ReductionCode &reduction_code) const
void mark_function_always_inline(llvm::Function *func)
Calculate approximate median and general quantiles, based on "Computing Extremely Accurate Quantiles ...
void reduceLoop(const ReductionCode &reduction_code) const
const size_t INTERP_THRESHOLD
llvm::Function * llvm_reduce_loop
void reduceOneEntryNoCollisionsIdx(const ReductionCode &reduction_code) const
bool hasKeylessHash() const
std::vector< std::string > CodeCacheKey
size_t get_slot_off_quad(const QueryMemoryDescriptor &query_mem_desc)
std::string cacheKey() const
size_t getEffectiveKeyWidth() const
std::string toString(const QueryDescriptionType &type)
std::unique_ptr< Function > ir_reduce_one_entry
bool g_enable_dynamic_watchdog
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co)
const std::vector< int64_t > target_init_vals_
void reduceOneAggregateSlot(Value *this_ptr1, Value *this_ptr2, Value *that_ptr1, Value *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const int64_t init_val, const int8_t chosen_bytes, Function *ir_reduce_one_entry) const
bool takes_float_argument(const TargetInfo &target_info)
bool g_enable_non_kernel_time_query_interrupt
Value * add(Args &&...args)
const Value * emit_checked_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
std::unique_ptr< Function > setup_reduce_one_entry_idx(ReductionCode *reduction_code)
int32_t(*)(int8_t *this_buff, const int8_t *that_buff, const int32_t start_entry_index, const int32_t end_entry_index, const int32_t that_entry_count, const void *this_qmd, const void *that_qmd, const void *serialized_varlen_buffer) FuncPtr
const QueryMemoryDescriptor query_mem_desc_
std::unique_ptr< Function > ir_is_empty
void translate_function(const Function *function, llvm::Function *llvm_function, const ReductionCode &reduction_code, const std::unordered_map< const Function *, llvm::Function * > &f)
void emit_aggregate_one_value(const std::string &agg_kind, Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
RUNTIME_EXPORT void serialized_varlen_buffer_sample(const void *serialized_varlen_buffer_handle, int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const int64_t init_val, const int64_t length_to_elems)
Value * emit_load_i32(Value *ptr, Function *function)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
const SQLTypeInfo get_compact_type(const TargetInfo &target)
bool blocksShareMemory() const
__device__ bool check_interrupt()
int8_t get_width_for_slot(const size_t target_slot_idx, const bool float_argument_input, const QueryMemoryDescriptor &query_mem_desc)
llvm::LLVMContext & context_
size_t get_byteoff_of_slot(const size_t slot_idx, const QueryMemoryDescriptor &query_mem_desc)
size_t advance_slot(const size_t j, const TargetInfo &target_info, const bool separate_varlen_storage)
void reduceOneCountDistinctSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
uint8_t check_interrupt_rt(const size_t sample_seed)
size_t getGroupbyColCount() const
const int32_t INTERRUPT_ERROR
size_t targetGroupbyIndicesSize() const
void generate_loop_body(For *for_loop, Function *ir_reduce_loop, Function *ir_reduce_one_entry_idx, Value *this_buff, Value *that_buff, Value *start_index, Value *that_entry_count, Value *this_qmd_handle, Value *that_qmd_handle, Value *serialized_varlen_buffer)
void emit_write_projection(Value *slot_pi8, Value *other_pi8, const int64_t init_val, const size_t chosen_bytes, Function *ir_reduce_one_entry)
void emit_aggregate_one_nullable_value(SQLAgg const sql_agg, Value *val_ptr, Value *other_ptr, const int64_t init_val, const size_t chosen_bytes, const TargetInfo &agg_info, Function *ir_reduce_one_entry)
ReductionCode codegen() const override
bool is_distinct_target(const TargetInfo &target_info)
std::string target_info_key(const TargetInfo &target_info)
OUTPUT transform(INPUT const &input, FUNC const &func)
std::unique_ptr< Function > ir_reduce_one_entry_idx
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
ReductionCode setup_functions_ir(const QueryDescriptionType hash_type)
std::unique_ptr< Function > setup_is_empty_entry(ReductionCode *reduction_code)
size_t getCountDistinctDescriptorsSize() const
void reduceOneEntryTargetsNoCollisions(Function *ir_reduce_one_entry, Value *this_targets_start_ptr, Value *that_targets_start_ptr) const
QueryDescriptionType getQueryDescriptionType() const
void set_module_shallow_copy(const std::unique_ptr< llvm::Module > &module, bool always_clone=false)
llvm::Type * llvm_type(const Type type, llvm::LLVMContext &ctx)
virtual ReductionCode codegen() const
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
RUNTIME_EXPORT void mode_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
std::string serialize_llvm_object(const T *llvm_obj)
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
RUNTIME_EXPORT void count_distinct_set_union_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
void finalizeReductionCode(ReductionCode &reduction_code, const llvm::Function *ir_is_empty, const llvm::Function *ir_reduce_one_entry, const llvm::Function *ir_reduce_one_entry_idx, const CodeCacheKey &key) const
std::unique_ptr< Function > create_function(const std::string name, const std::vector< Function::NamedArg > &arg_types, const Type ret_type, const bool always_inline)
std::string get_type_name() const
torch::Tensor f(torch::Tensor x, torch::Tensor W_target, torch::Tensor b_target)
const Value * iter() const
RUNTIME_EXPORT void approx_quantile_jit_rt(const int64_t new_set_handle, const int64_t old_set_handle, const void *that_qmd_handle, const void *this_qmd_handle, const int64_t target_logical_idx)
RUNTIME_EXPORT void get_group_value_reduction_rt(int8_t *groups_buffer, const int8_t *key, const uint32_t key_count, const void *this_qmd_handle, const int8_t *that_buff, const uint32_t that_entry_idx, const uint32_t that_entry_count, const uint32_t row_size_bytes, int64_t **buff_out, uint8_t *empty)
bool didOutputColumnar() const
std::unique_ptr< Function > setup_reduce_one_entry(ReductionCode *reduction_code, const QueryDescriptionType hash_type)
void isEmpty(const ReductionCode &reduction_code) const
Value * emit_load_i64(Value *ptr, Function *function)
const ColSlotContext & getColSlotContext() const
void reduceOneEntryBaseline(const ReductionCode &reduction_code) const
static std::shared_ptr< QueryEngine > getInstance()
const int32_t WATCHDOG_ERROR
Value * emit_load(Value *ptr, Type ptr_type, Function *function)
llvm::Function * create_llvm_function(const Function *function, CgenState *cgen_state)
ResultSetReductionJIT(const QueryMemoryDescriptor &query_mem_desc, const std::vector< TargetInfo > &targets, const std::vector< int64_t > &target_init_vals, const size_t executor_id)
HOST DEVICE bool get_notnull() const
void emit_aggregate_one_count(Value *val_ptr, Value *other_ptr, const size_t chosen_bytes, Function *ir_reduce_one_entry)
SQLTypeInfo get_elem_type() const
void reduceOneApproxQuantileSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
std::unique_ptr< Function > setup_reduce_loop(ReductionCode *reduction_code)
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
SQLAgg get_non_conditional_agg_type(SQLAgg const agg_type)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
std::string reductionKey() const
const Executor * getExecutor() const
void reduceOneModeSlot(Value *this_ptr1, Value *that_ptr1, const size_t target_logical_idx, Function *ir_reduce_one_entry) const
int32_t getTargetIdxForKey() const
const std::vector< TargetInfo > targets_