32 #include "../CudaMgr/CudaMgr.h"
33 #include "../Shared/checked_alloc.h"
34 #include "../Shared/funcannotations.h"
35 #include "../Utils/ChunkIter.h"
45 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
50 #include <string_view>
61 return min == 0 &&
max == -1;
65 out <<
"Hash Type = " << info.
hash_type_ <<
" min = " << info.
min
66 <<
" max = " << info.
max <<
" bucket = " << info.
bucket
67 <<
" has_nulls = " << info.
has_nulls <<
"\n";
80 out <<
"UnorderedSet";
83 out <<
"<Unkown Type>";
102 for (
auto target_expr : target_exprs) {
105 if (!agg_expr || agg_expr->get_aggtype() ==
kSAMPLE) {
107 if (ti.is_buffer()) {
109 }
else if (ti.is_geometry()) {
110 agg_count += ti.get_physical_coord_cols() * 2;
116 if (agg_expr && agg_expr->get_aggtype() ==
kAVG) {
131 if (!cd || !cd->isVirtualCol) {
139 for (
const auto& target_expr : ra_exe_unit.
target_exprs) {
149 const int64_t max_entry_count) {
175 const std::vector<InputTableInfo>& query_infos,
177 Executor* executor) {
183 expr, query_infos, executor, boost::make_optional(ra_exe_unit.
simple_quals));
184 switch (expr_range.getType()) {
186 if (expr_range.getIntMin() > expr_range.getIntMax()) {
191 expr_range.getIntMin(),
192 expr_range.getIntMax(),
193 expr_range.getBucket(),
194 expr_range.hasNulls()};
198 if (expr_range.getFpMin() > expr_range.getFpMax()) {
221 const int64_t baseline_threshold =
228 bool has_nulls{
false};
236 group_cardinality_estimation,
242 cardinality *= crt_col_cardinality;
243 if (col_range_info.has_nulls) {
248 if (!cardinality || cardinality > baseline_threshold) {
251 group_cardinality_estimation,
259 int64_t(cardinality),
265 group_cardinality_estimation,
279 group_cardinality_estimation,
286 return col_range_info;
289 const int64_t col_count =
291 int64_t max_entry_count = MAX_BUFFER_SIZE / (col_count *
sizeof(int64_t));
293 max_entry_count = std::min(max_entry_count, baseline_threshold);
296 if (groupby_expr_ti.is_string() && !col_range_info.bucket) {
299 const bool has_filters =
323 col_range_info.has_nulls};
326 return col_range_info;
338 col_range_info.has_nulls};
343 !col_range_info.bucket) {
348 col_range_info.has_nulls};
350 return col_range_info;
356 if (col_range_info.
bucket) {
357 crt_col_cardinality /= col_range_info.
bucket;
359 return static_cast<int64_t
>(crt_col_cardinality +
360 (1 + (col_range_info.
has_nulls ? 1 : 0)));
366 if (col_range_info.
min <= col_range_info.
max) {
367 size_t size = col_range_info.
max - col_range_info.
min;
368 if (col_range_info.
bucket) {
369 size /= col_range_info.
bucket;
371 if (size >= static_cast<size_t>(std::numeric_limits<int64_t>::max())) {
376 return static_cast<int64_t
>(size + 1);
383 #define LL_CONTEXT executor_->cgen_state_->context_
384 #define LL_BUILDER executor_->cgen_state_->ir_builder_
385 #define LL_BOOL(v) executor_->cgen_state_->llBool(v)
386 #define LL_INT(v) executor_->cgen_state_->llInt(v)
387 #define LL_FP(v) executor_->cgen_state_->llFp(v)
388 #define ROW_FUNC executor_->cgen_state_->row_func_
389 #define CUR_FUNC executor_->cgen_state_->current_func_
395 const std::vector<InputTableInfo>& query_infos,
396 std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
397 const std::optional<int64_t>& group_cardinality_estimation)
399 , ra_exe_unit_(ra_exe_unit)
400 , query_infos_(query_infos)
401 , row_set_mem_owner_(row_set_mem_owner)
402 , device_type_(device_type)
403 , group_cardinality_estimation_(group_cardinality_estimation) {
408 const auto& groupby_ti = groupby_expr->get_type_info();
409 if (groupby_ti.is_text_encoding_none()) {
410 throw std::runtime_error(
411 "Cannot group by string columns which are not dictionary encoded.");
413 if (groupby_ti.is_buffer()) {
414 throw std::runtime_error(
"Group by buffer not supported");
416 if (groupby_ti.is_geometry()) {
417 throw std::runtime_error(
"Group by geometry not supported");
423 const size_t shard_count)
const {
424 size_t device_count{0};
426 device_count =
executor_->cudaMgr()->getDeviceCount();
430 int64_t bucket{col_range_info.
bucket};
453 if (device_count < shard_count) {
454 bucket =
g_leaf_count ? std::max(device_count, static_cast<size_t>(1))
455 : std::min(device_count, shard_count - device_count);
457 bucket = shard_count * std::max(
g_leaf_count, static_cast<size_t>(1));
476 const std::vector<InputTableInfo>& query_infos,
477 const bool is_group_by,
478 Executor* executor) {
479 bool keyless{
true}, found{
false};
480 int32_t num_agg_expr{0};
482 for (
const auto target_expr : ra_exe_unit.
target_exprs) {
485 if (agg_info.is_agg) {
491 const auto arg_expr =
agg_arg(target_expr);
493 switch (agg_info.agg_kind) {
496 if (arg_expr && !arg_expr->get_type_info().get_notnull()) {
499 expr_range_info.hasNulls()) {
506 if (arg_expr && !arg_expr->get_type_info().get_notnull()) {
509 expr_range_info.hasNulls()) {
516 auto arg_ti = arg_expr->get_type_info();
518 arg_ti.set_notnull(
true);
520 if (!arg_ti.get_notnull()) {
523 !expr_range_info.hasNulls()) {
528 switch (expr_range_info.getType()) {
531 if (expr_range_info.getFpMax() < 0 || expr_range_info.getFpMin() > 0) {
536 if (expr_range_info.getIntMax() < 0 || expr_range_info.getIntMin() > 0) {
547 CHECK(agg_expr && agg_expr->get_arg());
548 const auto& arg_ti = agg_expr->get_arg()->get_type_info();
549 if (arg_ti.is_string() || arg_ti.is_buffer()) {
552 auto expr_range_info =
556 is_group_by || float_argument_input,
557 float_argument_input ?
sizeof(
float) : 8);
558 switch (expr_range_info.getType()) {
562 *
reinterpret_cast<const double*
>(may_alias_ptr(&init_max));
563 if (expr_range_info.getFpMax() < double_max) {
569 if (expr_range_info.getIntMax() < init_max) {
579 CHECK(agg_expr && agg_expr->get_arg());
580 const auto& arg_ti = agg_expr->get_arg()->get_type_info();
581 if (arg_ti.is_string() || arg_ti.is_buffer()) {
584 auto expr_range_info =
589 expr_range_info.hasNulls()) {
594 is_group_by || float_argument_input,
595 float_argument_input ?
sizeof(
float) : 8);
596 switch (expr_range_info.getType()) {
600 *
reinterpret_cast<const double*
>(may_alias_ptr(&init_min));
601 if (expr_range_info.getFpMin() > double_min) {
607 if (expr_range_info.getIntMin() > init_min) {
638 const std::vector<InputTableInfo>& query_infos,
641 Executor* executor) {
643 auto compute_bytes_per_group =
645 size_t effective_size_bytes = (bitmap_sz + 7) / 8;
646 const auto padded_size =
649 : effective_size_bytes;
650 return padded_size * sub_bitmap_count;
652 for (
size_t i = 0; i < ra_exe_unit.
target_exprs.size(); i++) {
656 CHECK(agg_info.is_agg);
660 if (arg_ti.is_text_encoding_none()) {
661 throw std::runtime_error(
662 "Strings must be dictionary-encoded for COUNT(DISTINCT).");
665 throw std::runtime_error(
"APPROX_COUNT_DISTINCT on arrays not supported yet");
668 throw std::runtime_error(
669 "APPROX_COUNT_DISTINCT on geometry columns not supported");
671 if (agg_info.is_distinct && arg_ti.is_geometry()) {
672 throw std::runtime_error(
"COUNT DISTINCT on geometry columns not supported");
675 auto arg_range_info =
676 arg_ti.is_fp() ? no_range_info
678 ra_exe_unit, query_infos, agg_expr->get_arg(), executor);
681 const auto& original_target_expr_ti = it->second;
682 if (arg_ti.is_integer() && original_target_expr_ti.get_type() ==
kDATE &&
686 auto is_date_value_not_encoded = [&original_target_expr_ti](int64_t date_val) {
687 if (original_target_expr_ti.get_comp_param() == 16) {
688 return date_val < INT16_MIN || date_val > INT16_MAX;
690 return date_val < INT32_MIN || date_val > INT32_MIN;
693 if (is_date_value_not_encoded(arg_range_info.min)) {
699 if (is_date_value_not_encoded(arg_range_info.max)) {
705 arg_range_info.bucket = 0;
710 int64_t bitmap_sz_bits{0};
712 const auto error_rate_expr = agg_expr->get_arg1();
713 if (error_rate_expr) {
714 CHECK(error_rate_expr->get_type_info().get_type() ==
kINT);
715 auto const error_rate =
718 CHECK_GE(error_rate->get_constval().intval, 1);
724 if (arg_range_info.isEmpty()) {
725 count_distinct_descriptors.emplace_back(
728 arg_range_info.bucket,
735 const auto sub_bitmap_count =
737 size_t worst_case_num_groups{1};
739 !(arg_ti.is_buffer() || arg_ti.is_geometry())) {
742 if (shared::is_any<kCOUNT, kCOUNT_IF>(agg_info.agg_kind)) {
748 const auto total_bytes_per_entry =
749 compute_bytes_per_group(bitmap_sz_bits, sub_bitmap_count, device_type);
750 const auto range_bucket = std::max(group_by_range_info.
bucket, (int64_t)1);
751 const auto maximum_num_groups =
752 (group_by_range_info.
max - group_by_range_info.
min + 1) / range_bucket;
753 const auto total_bitmap_bytes_for_groups =
754 total_bytes_per_entry * maximum_num_groups;
757 if (total_bitmap_bytes_for_groups >=
759 const auto agg_expr_max_entry_count =
760 arg_range_info.max - arg_range_info.min + 1;
761 int64_t max_agg_expr_table_cardinality{1};
763 bool (*)(
const Analyzer::ColumnVar*,
const Analyzer::ColumnVar*)>
766 for (
const auto cv : colvar_set) {
768 std::find_if(query_infos.begin(),
770 [&](
const auto& input_table_info) {
771 return input_table_info.table_key == cv->getTableKey();
773 int64_t cur_table_cardinality =
774 it != query_infos.end()
775 ?
static_cast<int64_t
>(it->info.getNumTuplesUpperBound())
777 max_agg_expr_table_cardinality =
778 std::max(max_agg_expr_table_cardinality, cur_table_cardinality);
779 worst_case_num_groups *= cur_table_cardinality;
781 auto has_valid_stat = [agg_expr_max_entry_count, maximum_num_groups]() {
782 return agg_expr_max_entry_count > 0 && maximum_num_groups > 0;
785 if (has_valid_stat()) {
789 const size_t unordered_set_threshold{2};
795 const auto bits_for_agg_entry = std::ceil(log(agg_expr_max_entry_count));
796 const auto bits_for_agg_table =
797 std::ceil(log(max_agg_expr_table_cardinality));
798 const auto avg_num_unique_entries_per_group =
799 std::ceil(max_agg_expr_table_cardinality / maximum_num_groups);
806 if ((bits_for_agg_entry - bits_for_agg_table) >= unordered_set_threshold ||
807 agg_expr_max_entry_count >= avg_num_unique_entries_per_group) {
810 throw std::runtime_error(
811 "Consider using approx_count_distinct operator instead of "
812 "count_distinct operator to lower the memory "
821 !(arg_ti.is_array() || arg_ti.is_geometry())) {
824 const size_t too_many_entries{100000000};
826 worst_case_num_groups > too_many_entries &&
829 "Detect too many input entries for set-based count distinct operator under "
832 count_distinct_descriptors.emplace_back(
835 arg_range_info.bucket,
845 return count_distinct_descriptors;
851 const bool allow_multifrag,
852 const size_t max_groups_buffer_entry_count,
853 const int8_t crt_min_byte_width,
855 const bool output_columnar_hint) {
859 bool sort_on_gpu_hint =
867 bool must_use_baseline_sort = shard_count;
871 max_groups_buffer_entry_count,
875 must_use_baseline_sort,
876 output_columnar_hint);
877 CHECK(query_mem_desc);
878 if (query_mem_desc->sortOnGpu() &&
880 align_to_int64(query_mem_desc->getEntryCount() *
sizeof(int32_t))) >
881 2 * 1024 * 1024 * 1024LL) {
882 must_use_baseline_sort =
true;
883 sort_on_gpu_hint =
false;
892 const bool allow_multifrag,
893 const size_t max_groups_buffer_entry_count,
894 const int8_t crt_min_byte_width,
895 const bool sort_on_gpu_hint,
897 const bool must_use_baseline_sort,
898 const bool output_columnar_hint) {
901 const bool threads_can_reuse_group_by_buffers =
911 const auto col_range_info =
913 col_range_info_nosharding.min,
914 col_range_info_nosharding.max,
916 col_range_info_nosharding.has_nulls};
920 const auto keyless_info =
931 (col_range_info.max - col_range_info.min) /
932 std::max(col_range_info.bucket, int64_t(1)) >
950 max_groups_buffer_entry_count,
952 count_distinct_descriptors,
953 must_use_baseline_sort,
954 output_columnar_hint,
956 threads_can_reuse_group_by_buffers);
958 LOG(
WARNING) << e.what() <<
" Disabling Streaming Top N.";
969 max_groups_buffer_entry_count,
971 count_distinct_descriptors,
972 must_use_baseline_sort,
973 output_columnar_hint,
975 threads_can_reuse_group_by_buffers);
980 const std::list<Analyzer::OrderEntry>& order_entries) {
981 if (order_entries.size() > 1) {
984 for (
const auto& order_entry : order_entries) {
988 if (!dynamic_cast<Analyzer::AggExpr*>(target_expr)) {
993 if (agg_expr->get_is_distinct() || agg_expr->get_aggtype() ==
kAVG ||
994 agg_expr->get_aggtype() ==
kMIN || agg_expr->get_aggtype() ==
kMAX ||
998 if (agg_expr->get_arg()) {
1000 if (arg_ti.is_fp()) {
1003 auto expr_range_info =
1008 expr_range_info.has_nulls) &&
1009 order_entry.is_desc == order_entry.nulls_first) {
1013 const auto& target_ti = target_expr->get_type_info();
1014 CHECK(!target_ti.is_buffer());
1015 if (!target_ti.is_integer()) {
1023 llvm::BasicBlock* sc_false,
1028 CHECK(filter_result);
1030 bool can_return_error =
false;
1031 llvm::BasicBlock* filter_false{
nullptr};
1053 llvm::Value* old_total_matched_val{
nullptr};
1055 old_total_matched_val =
1056 LL_BUILDER.CreateAtomicRMW(llvm::AtomicRMWInst::Add,
1059 #
if LLVM_VERSION_MAJOR > 12
1062 llvm::AtomicOrdering::Monotonic);
1064 old_total_matched_val =
LL_BUILDER.CreateLoad(
1065 total_matched_ptr->getType()->getPointerElementType(), total_matched_ptr);
1071 LL_BUILDER.CreateStore(old_total_matched_val, old_total_matched_ptr);
1074 auto agg_out_ptr_w_idx =
codegenGroupBy(query_mem_desc, co, filter_cfg);
1080 filter_cfg.setChainToNext();
1085 varlen_output_buffer,
1093 llvm::Value* nullcheck_cond{
nullptr};
1095 nullcheck_cond =
LL_BUILDER.CreateICmpSGE(std::get<1>(agg_out_ptr_w_idx),
1099 std::get<0>(agg_out_ptr_w_idx),
1100 llvm::ConstantPointerNull::get(
1104 nullcheck_cond,
executor_,
false,
"groupby_nullcheck", &filter_cfg,
false);
1106 varlen_output_buffer,
1113 can_return_error =
true;
1123 code_generator.
posArg(
nullptr),
1129 std::stack<llvm::BasicBlock*> array_loops;
1132 auto arg_it =
ROW_FUNC->arg_begin();
1133 std::vector<llvm::Value*> agg_out_vec;
1135 agg_out_vec.push_back(&*arg_it++);
1150 }
else if (sc_false) {
1151 const auto saved_insert_block =
LL_BUILDER.GetInsertBlock();
1154 LL_BUILDER.SetInsertPoint(saved_insert_block);
1157 return can_return_error;
1161 llvm::Value* groups_buffer,
1175 : query_mem_desc.
getRowSize() /
sizeof(int64_t);
1179 CHECK_GE(only_order_entry.tle_no,
int(1));
1180 const size_t target_idx = only_order_entry.tle_no - 1;
1183 const auto chosen_bytes =
1185 auto order_entry_lv =
executor_->cgen_state_->castToTypeIn(
1186 code_generator.
codegen(order_entry_expr,
true, co).front(), chosen_bytes * 8);
1189 std::string fname =
"get_bin_from_k_heap";
1190 const auto& oe_ti = order_entry_expr->get_type_info();
1191 llvm::Value* null_key_lv =
nullptr;
1192 if (oe_ti.is_integer() || oe_ti.is_decimal() || oe_ti.is_time()) {
1193 const size_t bit_width = order_entry_lv->getType()->getIntegerBitWidth();
1194 switch (bit_width) {
1206 CHECK(oe_ti.is_fp());
1207 if (order_entry_lv->getType()->isDoubleTy()) {
1212 fname += order_entry_lv->getType()->isDoubleTy() ?
"_double" :
"_float";
1214 const auto key_slot_idx =
1222 LL_BOOL(only_order_entry.is_desc),
1223 LL_BOOL(!order_entry_expr->get_type_info().get_notnull()),
1224 LL_BOOL(only_order_entry.nulls_first),
1229 const auto output_buffer_entry_count_lv =
1230 LL_BUILDER.CreateLoad(arg->getType()->getPointerElementType(), arg);
1232 const auto group_expr_lv =
1233 LL_BUILDER.CreateLoad(arg->getType()->getPointerElementType(), arg);
1234 std::vector<llvm::Value*>
args{groups_buffer,
1235 output_buffer_entry_count_lv,
1237 code_generator.
posArg(
nullptr)};
1239 const auto columnar_output_offset =
1241 return columnar_output_offset;
1253 auto arg_it =
ROW_FUNC->arg_begin();
1254 auto groups_buffer = arg_it++;
1256 std::stack<llvm::BasicBlock*> array_loops;
1261 return std::make_tuple(
1265 return std::make_tuple(
1278 : query_mem_desc.
getRowSize() /
sizeof(int64_t);
1284 llvm::Value* group_key =
nullptr;
1285 llvm::Value* key_size_lv =
nullptr;
1296 col_width_size ==
sizeof(int32_t)
1304 int32_t subkey_idx = 0;
1307 const auto col_range_info =
1309 const auto translated_null_value =
static_cast<int64_t
>(
1314 (col_range_info.bucket ? col_range_info.bucket : 1));
1316 const bool col_has_nulls =
1321 : col_range_info.has_nulls)
1324 const auto group_expr_lvs =
1325 executor_->groupByColumnCodegen(group_expr.get(),
1329 translated_null_value,
1333 const auto group_expr_lv = group_expr_lvs.translated_value;
1340 group_expr_lvs.original_value,
1347 group_key->getType()->getScalarType()->getPointerElementType(),
1356 &*groups_buffer, group_key, key_size_lv, query_mem_desc, row_size_quad);
1368 return std::make_tuple(
nullptr,
nullptr);
1378 auto arg_it =
ROW_FUNC->arg_begin();
1380 auto varlen_output_buffer = arg_it++;
1381 CHECK(varlen_output_buffer->getType() == llvm::Type::getInt64PtrTy(
LL_CONTEXT));
1382 return varlen_output_buffer;
1385 std::tuple<llvm::Value*, llvm::Value*>
1389 llvm::Value* groups_buffer,
1390 llvm::Value* group_expr_lv_translated,
1391 llvm::Value* group_expr_lv_original,
1392 const int32_t row_size_quad) {
1396 ?
"get_columnar_group_bin_offset"
1397 :
"get_group_value_fast"};
1399 get_group_fn_name +=
"_keyless";
1404 get_group_fn_name +=
"_semiprivate";
1406 std::vector<llvm::Value*> get_group_fn_args{&*groups_buffer,
1407 &*group_expr_lv_translated};
1408 if (group_expr_lv_original && get_group_fn_name ==
"get_group_value_fast" &&
1410 get_group_fn_name +=
"_with_original_key";
1411 get_group_fn_args.push_back(group_expr_lv_original);
1417 get_group_fn_args.push_back(
LL_INT(row_size_quad));
1421 get_group_fn_args.push_back(
LL_INT(row_size_quad));
1425 get_group_fn_args.push_back(warp_idx);
1429 if (get_group_fn_name ==
"get_columnar_group_bin_offset") {
1430 return std::make_tuple(&*groups_buffer,
1431 emitCall(get_group_fn_name, get_group_fn_args));
1433 return std::make_tuple(
emitCall(get_group_fn_name, get_group_fn_args),
nullptr);
1437 llvm::Value* groups_buffer,
1438 llvm::Value* group_key,
1439 llvm::Value* key_size_lv,
1441 const int32_t row_size_quad) {
1448 LL_BUILDER.CreateCall(perfect_hash_func, std::vector<llvm::Value*>{group_key});
1452 const std::string set_matching_func_name{
1453 "set_matching_group_value_perfect_hash_columnar"};
1454 const std::vector<llvm::Value*> set_matching_func_arg{
1461 emitCall(set_matching_func_name, set_matching_func_arg);
1463 return std::make_tuple(groups_buffer, hash_lv);
1466 return std::make_tuple(
emitCall(
"get_matching_group_value_perfect_hash_keyless",
1467 {groups_buffer, hash_lv,
LL_INT(row_size_quad)}),
1470 return std::make_tuple(
1472 "get_matching_group_value_perfect_hash",
1473 {groups_buffer, hash_lv, group_key, key_size_lv,
LL_INT(row_size_quad)}),
1479 std::tuple<llvm::Value*, llvm::Value*>
1482 llvm::Value* groups_buffer,
1483 llvm::Value* group_key,
1484 llvm::Value* key_size_lv,
1486 const size_t key_width,
1487 const int32_t row_size_quad) {
1489 if (group_key->getType() != llvm::Type::getInt64PtrTy(
LL_CONTEXT)) {
1490 CHECK(key_width ==
sizeof(int32_t));
1494 std::vector<llvm::Value*> func_args{
1499 LL_INT(static_cast<int32_t>(key_width))};
1500 std::string func_name{
"get_group_value"};
1502 func_name +=
"_columnar_slot";
1504 func_args.push_back(
LL_INT(row_size_quad));
1507 func_name +=
"_with_watchdog";
1510 return std::make_tuple(groups_buffer,
emitCall(func_name, func_args));
1512 return std::make_tuple(
emitCall(func_name, func_args),
nullptr);
1519 auto ft = llvm::FunctionType::get(
1523 auto key_hash_func = llvm::Function::Create(ft,
1524 llvm::Function::ExternalLinkage,
1527 executor_->cgen_state_->helper_functions_.push_back(key_hash_func);
1529 auto& key_buff_arg = *key_hash_func->args().begin();
1530 llvm::Value* key_buff_lv = &key_buff_arg;
1531 auto bb = llvm::BasicBlock::Create(
LL_CONTEXT,
"entry", key_hash_func);
1532 llvm::IRBuilder<> key_hash_func_builder(bb);
1534 std::vector<int64_t> cardinalities;
1536 auto col_range_info =
1543 auto* gep = key_hash_func_builder.CreateGEP(
1544 key_buff_lv->getType()->getScalarType()->getPointerElementType(),
1548 key_hash_func_builder.CreateLoad(gep->getType()->getPointerElementType(), gep);
1549 auto col_range_info =
1552 key_hash_func_builder.CreateSub(key_comp_lv,
LL_INT(col_range_info.min));
1553 if (col_range_info.bucket) {
1555 key_hash_func_builder.CreateSDiv(crt_term_lv,
LL_INT(col_range_info.bucket));
1557 for (
size_t prev_dim_idx = 0; prev_dim_idx < dim_idx; ++prev_dim_idx) {
1558 crt_term_lv = key_hash_func_builder.CreateMul(crt_term_lv,
1559 LL_INT(cardinalities[prev_dim_idx]));
1561 hash_lv = key_hash_func_builder.CreateAdd(hash_lv, crt_term_lv);
1564 key_hash_func_builder.CreateRet(
1566 return key_hash_func;
1571 llvm::Value* target) {
1573 const auto& agg_type = agg_info.
sql_type;
1574 const size_t chosen_bytes = agg_type.
get_size();
1576 bool need_conversion{
false};
1577 llvm::Value* arg_null{
nullptr};
1578 llvm::Value* agg_null{
nullptr};
1579 llvm::Value* target_to_cast{target};
1580 if (arg_type.
is_fp()) {
1581 arg_null =
executor_->cgen_state_->inlineFpNull(arg_type);
1582 if (agg_type.is_fp()) {
1583 agg_null =
executor_->cgen_state_->inlineFpNull(agg_type);
1584 if (!static_cast<llvm::ConstantFP*>(arg_null)->isExactlyValue(
1585 static_cast<llvm::ConstantFP*>(agg_null)->getValueAPF())) {
1586 need_conversion =
true;
1593 arg_null =
executor_->cgen_state_->inlineIntNull(arg_type);
1594 if (agg_type.is_fp()) {
1595 agg_null =
executor_->cgen_state_->inlineFpNull(agg_type);
1596 need_conversion =
true;
1597 target_to_cast =
executor_->castToFP(target, arg_type, agg_type);
1599 agg_null =
executor_->cgen_state_->inlineIntNull(agg_type);
1600 if ((static_cast<llvm::ConstantInt*>(arg_null)->getBitWidth() !=
1601 static_cast<llvm::ConstantInt*>(agg_null)->getBitWidth()) ||
1602 (static_cast<llvm::ConstantInt*>(arg_null)->getValue() !=
1603 static_cast<llvm::ConstantInt*>(agg_null)->getValue())) {
1604 need_conversion =
true;
1608 if (need_conversion) {
1609 auto cmp = arg_type.
is_fp() ?
LL_BUILDER.CreateFCmpOEQ(target, arg_null)
1614 executor_->cgen_state_->castToTypeIn(target_to_cast, chosen_bytes << 3));
1626 const auto window_func_context =
1631 : query_mem_desc.
getRowSize() /
sizeof(int64_t);
1632 auto arg_it =
ROW_FUNC->arg_begin();
1633 auto groups_buffer = arg_it++;
1636 window_func_context, code_generator.
posArg(
nullptr));
1637 const auto pos_in_window =
1639 llvm::Value* entry_count_lv =
1641 std::vector<llvm::Value*>
args{
1642 &*groups_buffer, entry_count_lv, pos_in_window, code_generator.
posArg(
nullptr)};
1644 const auto columnar_output_offset =
1651 auto arg_it =
ROW_FUNC->arg_begin();
1652 auto groups_buffer = arg_it++;
1657 const std::tuple<llvm::Value*, llvm::Value*>& agg_out_ptr_w_idx_in,
1658 llvm::Value* varlen_output_buffer,
1659 const std::vector<llvm::Value*>& agg_out_vec,
1665 auto agg_out_ptr_w_idx = agg_out_ptr_w_idx_in;
1668 const bool is_group_by = std::get<0>(agg_out_ptr_w_idx);
1669 bool can_return_error =
false;
1671 CHECK(agg_out_vec.empty());
1673 CHECK(!agg_out_vec.empty());
1678 llvm::Value* output_buffer_byte_stream{
nullptr};
1679 llvm::Value* out_row_idx{
nullptr};
1682 output_buffer_byte_stream =
LL_BUILDER.CreateBitCast(
1683 std::get<0>(agg_out_ptr_w_idx),
1684 llvm::PointerType::get(llvm::Type::getInt8Ty(
LL_CONTEXT), 0));
1685 output_buffer_byte_stream->setName(
"out_buff_b_stream");
1686 CHECK(std::get<1>(agg_out_ptr_w_idx));
1687 out_row_idx =
LL_BUILDER.CreateZExt(std::get<1>(agg_out_ptr_w_idx),
1689 out_row_idx->setName(
"out_row_idx");
1698 target_builder(target_expr,
executor_, query_mem_desc, co);
1708 output_buffer_byte_stream,
1710 varlen_output_buffer,
1713 return can_return_error;
1720 llvm::Value* output_buffer_byte_stream,
1721 llvm::Value* out_row_idx,
1722 const std::tuple<llvm::Value*, llvm::Value*>& agg_out_ptr_w_idx,
1724 const size_t chosen_bytes,
1725 const size_t agg_out_off,
1726 const size_t target_idx) {
1728 llvm::Value* agg_col_ptr{
nullptr};
1734 CHECK(chosen_bytes == 1 || chosen_bytes == 2 || chosen_bytes == 4 ||
1736 CHECK(output_buffer_byte_stream);
1740 auto out_per_col_byte_idx =
1742 LL_BUILDER.CreateShl(out_row_idx, __lzcnt(chosen_bytes) - 1);
1744 LL_BUILDER.CreateShl(out_row_idx, __builtin_ffs(chosen_bytes) - 1);
1746 auto byte_offset =
LL_BUILDER.CreateAdd(out_per_col_byte_idx,
1747 LL_INT(static_cast<int64_t>(col_off)));
1748 byte_offset->setName(
"out_byte_off_target_" +
std::to_string(target_idx));
1750 output_buffer_byte_stream->getType()->getScalarType()->getPointerElementType(),
1751 output_buffer_byte_stream,
1756 agg_col_ptr->setName(
"out_ptr_target_" +
std::to_string(target_idx));
1758 auto const col_off_in_bytes = query_mem_desc.
getColOffInBytes(agg_out_off);
1759 auto const col_off = col_off_in_bytes / chosen_bytes;
1760 auto const col_rem = col_off_in_bytes % chosen_bytes;
1761 CHECK_EQ(col_rem, 0u) << col_off_in_bytes <<
" % " << chosen_bytes;
1762 CHECK(std::get<1>(agg_out_ptr_w_idx));
1764 std::get<1>(agg_out_ptr_w_idx),
1768 std::get<0>(agg_out_ptr_w_idx),
1771 bit_cast->getType()->getScalarType()->getPointerElementType(),
1777 auto const col_off = col_off_in_bytes / chosen_bytes;
1778 auto const col_rem = col_off_in_bytes % chosen_bytes;
1779 CHECK_EQ(col_rem, 0u) << col_off_in_bytes <<
" % " << chosen_bytes;
1781 std::get<0>(agg_out_ptr_w_idx),
1784 bit_cast->getType()->getScalarType()->getPointerElementType(),
1798 auto estimator_comp_count_lv =
LL_INT(static_cast<int32_t>(estimator_arg.size()));
1800 estimator_comp_count_lv);
1801 int32_t subkey_idx = 0;
1802 for (
const auto& estimator_arg_comp : estimator_arg) {
1803 const auto estimator_arg_comp_lvs =
1804 executor_->groupByColumnCodegen(estimator_arg_comp.get(),
1812 CHECK(!estimator_arg_comp_lvs.original_value);
1813 const auto estimator_arg_comp_lv = estimator_arg_comp_lvs.translated_value;
1816 estimator_arg_comp_lv,
1818 estimator_key_lv->getType()->getScalarType()->getPointerElementType(),
1824 const auto key_bytes =
LL_BUILDER.CreateBitCast(estimator_key_lv, int8_ptr_ty);
1825 const auto estimator_comp_bytes_lv =
1826 LL_INT(static_cast<int32_t>(estimator_arg.size() *
sizeof(int64_t)));
1827 const auto bitmap_size_lv =
1830 {bitmap, &*bitmap_size_lv, key_bytes, &*estimator_comp_bytes_lv});
1839 const int64_t skip_val) {
1840 if (val != skip_val) {
1852 auto* mode_map =
reinterpret_cast<AggMode*
>(*agg);
1857 const size_t target_idx,
1859 std::vector<llvm::Value*>& agg_args,
1864 const auto& arg_ti =
1866 if (arg_ti.is_fp()) {
1867 agg_args.back() =
executor_->cgen_state_->ir_builder_.CreateBitCast(
1870 const auto& count_distinct_descriptor =
1875 agg_args.push_back(
LL_INT(int32_t(count_distinct_descriptor.bitmap_sz_bits)));
1879 agg_args.push_back(base_dev_addr);
1880 agg_args.push_back(base_host_addr);
1881 emitCall(
"agg_approximate_count_distinct_gpu", agg_args);
1883 emitCall(
"agg_approximate_count_distinct", agg_args);
1887 std::string agg_fname{
"agg_count_distinct"};
1889 agg_fname +=
"_bitmap";
1890 agg_args.push_back(
LL_INT(count_distinct_descriptor.min_val));
1891 agg_args.push_back(
LL_INT(count_distinct_descriptor.bucket_size));
1893 if (agg_info.skip_null_val) {
1894 auto null_lv =
executor_->cgen_state_->castToTypeIn(
1896 ?
static_cast<llvm::Value*
>(
executor_->cgen_state_->inlineFpNull(arg_ti))
1897 : static_cast<llvm::Value*>(
executor_->cgen_state_->inlineIntNull(arg_ti))),
1899 null_lv =
executor_->cgen_state_->ir_builder_.CreateBitCast(
1901 agg_fname +=
"_skip_val";
1902 agg_args.push_back(null_lv);
1906 agg_fname +=
"_gpu";
1909 agg_args.push_back(base_dev_addr);
1910 agg_args.push_back(base_host_addr);
1911 agg_args.push_back(
LL_INT(int64_t(count_distinct_descriptor.sub_bitmap_count)));
1913 count_distinct_descriptor.bitmapPaddedSizeBytes() %
1914 count_distinct_descriptor.sub_bitmap_count);
1915 agg_args.push_back(
LL_INT(int64_t(count_distinct_descriptor.bitmapPaddedSizeBytes() /
1916 count_distinct_descriptor.sub_bitmap_count)));
1921 executor_->cgen_state_->emitExternalCall(
1922 agg_fname, llvm::Type::getVoidTy(
LL_CONTEXT), agg_args);
1927 const size_t target_idx,
1929 std::vector<llvm::Value*>& agg_args,
1935 llvm::BasicBlock *calc, *skip{
nullptr};
1941 auto* cs =
executor_->cgen_state_.get();
1942 auto& irb = cs->ir_builder_;
1944 auto*
const null_value = cs->castToTypeIn(cs->inlineNull(arg_ti), 64);
1945 auto*
const skip_cond = arg_ti.is_fp()
1946 ? irb.CreateFCmpOEQ(agg_args.back(), null_value)
1947 : irb.CreateICmpEQ(agg_args.back(), null_value);
1948 calc = llvm::BasicBlock::Create(cs->context_,
"calc_approx_quantile");
1949 skip = llvm::BasicBlock::Create(cs->context_,
"skip_approx_quantile");
1950 irb.CreateCondBr(skip_cond, skip, calc);
1951 cs->current_func_->getBasicBlockList().push_back(calc);
1952 irb.SetInsertPoint(calc);
1954 if (!arg_ti.is_fp()) {
1956 agg_args.back() =
executor_->castToFP(agg_args.back(), arg_ti, agg_info.sql_type);
1958 cs->emitExternalCall(
1959 "agg_approx_quantile", llvm::Type::getVoidTy(cs->context_), agg_args);
1962 cs->current_func_->getBasicBlockList().push_back(skip);
1963 irb.SetInsertPoint(skip);
1969 std::vector<llvm::Value*>& agg_args,
1975 llvm::BasicBlock *calc, *skip{
nullptr};
1980 bool const is_fp = arg_ti.is_fp();
1981 auto* cs =
executor_->cgen_state_.get();
1982 auto& irb = cs->ir_builder_;
1984 auto*
const null_value =
1985 is_fp ? cs->inlineNull(arg_ti) : cs->castToTypeIn(cs->inlineNull(arg_ti), 64);
1986 auto*
const skip_cond = is_fp ? irb.CreateFCmpOEQ(agg_args.back(), null_value)
1987 : irb.CreateICmpEQ(agg_args.back(), null_value);
1988 calc = llvm::BasicBlock::Create(cs->context_,
"calc_mode");
1989 skip = llvm::BasicBlock::Create(cs->context_,
"skip_mode");
1990 irb.CreateCondBr(skip_cond, skip, calc);
1991 cs->current_func_->getBasicBlockList().push_back(calc);
1992 irb.SetInsertPoint(calc);
1995 auto*
const int_type =
get_int_type(8 * arg_ti.get_size(), cs->context_);
1996 agg_args.back() = irb.CreateBitCast(agg_args.back(), int_type);
1999 cs->emitExternalCall(
"agg_mode_func", llvm::Type::getVoidTy(cs->context_), agg_args);
2002 cs->current_func_->getBasicBlockList().push_back(skip);
2003 irb.SetInsertPoint(skip);
2016 return LL_BUILDER.CreateLoad(gep->getType()->getPointerElementType(), gep);
2031 if (target_ti.is_buffer() &&
2032 !
executor_->plan_state_->isLazyFetchColumn(target_expr)) {
2033 const auto target_lvs =
2034 agg_expr ? code_generator.
codegen(agg_expr->get_arg(),
true, co)
2036 target_expr, !
executor_->plan_state_->allow_lazy_fetch_, co);
2037 if (!func_expr && !arr_expr) {
2040 if (target_ti.is_text_encoding_none()) {
2041 CHECK_EQ(
size_t(3), target_lvs.size());
2042 return {target_lvs[1], target_lvs[2]};
2044 CHECK(target_ti.is_array());
2045 CHECK_EQ(
size_t(1), target_lvs.size());
2046 CHECK(!agg_expr || agg_expr->get_aggtype() ==
kSAMPLE);
2050 const auto& elem_ti = target_ti.get_elem_type();
2052 executor_->cgen_state_->emitExternalCall(
2055 {target_lvs.front(), code_generator.
posArg(target_expr)}),
2056 executor_->cgen_state_->emitExternalCall(
2059 {target_lvs.front(),
2060 code_generator.
posArg(target_expr),
2064 throw std::runtime_error(
2065 "Using array[] operator as argument to an aggregate operator is not "
2068 CHECK(func_expr || arr_expr);
2069 if (dynamic_cast<const Analyzer::FunctionOper*>(target_expr)) {
2070 CHECK_EQ(
size_t(1), target_lvs.size());
2071 const auto prefix = target_ti.get_buffer_name();
2072 CHECK(target_ti.is_array() || target_ti.is_text_encoding_none());
2073 const auto target_lv =
LL_BUILDER.CreateLoad(
2074 target_lvs[0]->getType()->getPointerElementType(), target_lvs[0]);
2078 const auto i8p_ty = llvm::PointerType::get(
2080 const auto ptr =
LL_BUILDER.CreatePointerCast(
2081 LL_BUILDER.CreateExtractValue(target_lv, 0), i8p_ty);
2082 const auto size =
LL_BUILDER.CreateExtractValue(target_lv, 1);
2083 const auto null_flag =
LL_BUILDER.CreateExtractValue(target_lv, 2);
2084 const auto nullcheck_ok_bb =
2086 const auto nullcheck_fail_bb = llvm::BasicBlock::Create(
2090 const auto nullcheck =
LL_BUILDER.CreateICmpEQ(
2091 null_flag,
executor_->cgen_state_->llInt(static_cast<int8_t>(1)));
2092 LL_BUILDER.CreateCondBr(nullcheck, nullcheck_fail_bb, nullcheck_ok_bb);
2097 auto result_phi =
LL_BUILDER.CreatePHI(i8p_ty, 2, prefix +
"_ptr_return");
2098 result_phi->addIncoming(ptr, nullcheck_ok_bb);
2099 const auto null_arr_sentinel =
LL_BUILDER.CreateIntToPtr(
2100 executor_->cgen_state_->llInt(static_cast<int8_t>(0)), i8p_ty);
2101 result_phi->addIncoming(null_arr_sentinel, nullcheck_fail_bb);
2103 executor_->cgen_state_->emitExternalCall(
2104 "register_buffer_with_executor_rsm",
2105 llvm::Type::getVoidTy(
executor_->cgen_state_->context_),
2108 LL_BUILDER.SetInsertPoint(nullcheck_fail_bb);
2112 return {result_phi, size};
2114 CHECK_EQ(
size_t(2), target_lvs.size());
2115 return {target_lvs[0], target_lvs[1]};
2118 if (target_ti.is_geometry() &&
2119 !
executor_->plan_state_->isLazyFetchColumn(target_expr)) {
2120 auto generate_coord_lvs =
2121 [&](
auto* selected_target_expr,
2122 bool const fetch_columns) -> std::vector<llvm::Value*> {
2123 const auto target_lvs =
2124 code_generator.
codegen(selected_target_expr, fetch_columns, co);
2125 if (dynamic_cast<const Analyzer::GeoOperator*>(target_expr) &&
2132 if (geo_uoper || geo_binoper) {
2134 CHECK_EQ(2 * static_cast<size_t>(target_ti.get_physical_coord_cols()),
2138 CHECK_EQ(static_cast<size_t>(target_ti.get_physical_coord_cols()),
2144 std::vector<llvm::Value*> coords;
2146 for (
const auto& target_lv : target_lvs) {
2152 const size_t elem_sz = ctr == 0 ? 1 : 4;
2154 int32_t fixlen = -1;
2155 if (target_ti.get_type() ==
kPOINT) {
2158 const auto coords_cd =
executor_->getPhysicalColumnDescriptor(col_var, 1);
2159 if (coords_cd && coords_cd->columnType.get_type() ==
kARRAY) {
2160 fixlen = coords_cd->columnType.get_size();
2165 coords.push_back(
executor_->cgen_state_->emitExternalCall(
2166 "fast_fixlen_array_buff",
2168 {target_lv, code_generator.
posArg(selected_target_expr)}));
2169 auto fixed_len_lv =
executor_->cgen_state_->emitExternalCall(
2170 "determine_fixed_array_len",
2172 {target_lv,
executor_->cgen_state_->llInt(int64_t(fixlen))});
2173 coords.push_back(fixed_len_lv);
2176 coords.push_back(
executor_->cgen_state_->emitExternalCall(
2179 {target_lv, code_generator.
posArg(selected_target_expr)}));
2180 coords.push_back(
executor_->cgen_state_->emitExternalCall(
2184 code_generator.
posArg(selected_target_expr),
2191 return generate_coord_lvs(agg_expr->get_arg(),
true);
2193 return generate_coord_lvs(target_expr,
2194 !
executor_->plan_state_->allow_lazy_fetch_);
2198 bool fetch_column = !
executor_->plan_state_->allow_lazy_fetch_;
2199 return agg_expr ? code_generator.codegen(agg_expr->get_arg(),
true, co)
2200 : code_generator.codegen(target_expr, fetch_column, co);
2204 const std::vector<llvm::Value*>&
args) {
2206 return executor_->cgen_state_->emitCall(fname, args);
2211 auto zero_const = llvm::ConstantInt::get(retCode->getType(), 0,
true);
2212 auto rc_check_condition =
executor_->cgen_state_->ir_builder_.CreateICmp(
2213 llvm::ICmpInst::ICMP_EQ, retCode, zero_const);
2215 executor_->cgen_state_->emitErrorCheck(rc_check_condition, retCode,
"rc");
2232 const auto grouped_col_expr =
2234 if (!grouped_col_expr) {
2237 const auto& column_key = grouped_col_expr->
getColumnKey();
2238 if (column_key.table_id <= 0) {
2242 {column_key.db_id, column_key.table_id});
2243 if (td->shardedColumnId == column_key.column_id) {
RUNTIME_EXPORT void agg_approx_quantile(int64_t *agg, const double val)
const Analyzer::Expr * agg_arg(const Analyzer::Expr *expr)
std::vector< Analyzer::Expr * > target_exprs
SqlWindowFunctionKind getKind() const
ExecutorDeviceType device_type
size_t g_watchdog_baseline_max_groups
bool constrained_not_null(const Analyzer::Expr *expr, const std::list< std::shared_ptr< Analyzer::Expr >> &quals)
robin_hood::unordered_set< int64_t > CountDistinctSet
bool gpuCanHandleOrderEntries(const std::list< Analyzer::OrderEntry > &order_entries)
static int64_t getBucketedCardinality(const ColRangeInfo &col_range_info)
llvm::Value * getAdditionalLiteral(const int32_t off)
ColRangeInfo get_expr_range_info(const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &query_infos, const Analyzer::Expr *expr, Executor *executor)
llvm::BasicBlock * cond_false_
llvm::Value * codegenAggColumnPtr(llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const QueryMemoryDescriptor &query_mem_desc, const size_t chosen_bytes, const size_t agg_out_off, const size_t target_idx)
: returns the pointer to where the aggregation should be stored.
HOST DEVICE int get_size() const
size_t getEntryCount() const
static bool colvar_comp(const ColumnVar *l, const ColumnVar *r)
RUNTIME_EXPORT void agg_count_distinct(int64_t *agg, const int64_t val)
int hll_size_for_rate(const int err_percent)
bool codegen(llvm::Value *filter_result, llvm::BasicBlock *sc_false, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const GpuSharedMemoryContext &gpu_smem_context)
boost::multiprecision::number< boost::multiprecision::cpp_int_backend< 64, 64, boost::multiprecision::signed_magnitude, boost::multiprecision::checked, void >> checked_int64_t
bool is_column_range_too_big_for_perfect_hash(const ColRangeInfo &col_range_info, const int64_t max_entry_count)
void collect_column_var(std::set< const ColumnVar *, bool(*)(const ColumnVar *, const ColumnVar *)> &colvar_set, bool include_agg) const override
KeylessInfo get_keyless_info(const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &query_infos, const bool is_group_by, Executor *executor)
std::unique_ptr< QueryMemoryDescriptor > initQueryMemoryDescriptorImpl(const bool allow_multifrag, const size_t max_groups_buffer_entry_count, const int8_t crt_min_byte_width, const bool sort_on_gpu_hint, RenderInfo *render_info, const bool must_use_baseline_sort, const bool output_columnar_hint)
void codegenMode(const size_t target_idx, const Analyzer::Expr *target_expr, std::vector< llvm::Value * > &agg_args, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type)
Streaming Top N algorithm.
bool mustUseBaselineSort() const
std::ostream & operator<<(std::ostream &os, const SessionInfo &session_info)
void mark_function_always_inline(llvm::Function *func)
ColRangeInfo getColRangeInfo()
bool hasVarlenOutput() const
QueryDescriptionType hash_type_
llvm::Value * posArg(const Analyzer::Expr *) const
const TableDescriptor * get_metadata_for_table(const ::shared::TableKey &table_key, bool populate_fragmenter)
llvm::Value * emitCall(const std::string &fname, const std::vector< llvm::Value * > &args)
bool hasKeylessHash() const
int64_t get_agg_initial_val(const SQLAgg agg, const SQLTypeInfo &ti, const bool enable_compaction, const unsigned min_byte_width_to_compact)
llvm::Value * codegenVarlenOutputBuffer(const QueryMemoryDescriptor &query_mem_desc)
static std::unique_ptr< QueryMemoryDescriptor > init(const Executor *executor, const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &query_infos, const ColRangeInfo &col_range_info, const KeylessInfo &keyless_info, const bool allow_multifrag, const ExecutorDeviceType device_type, const int8_t crt_min_byte_width, const bool sort_on_gpu_hint, const size_t shard_count, const size_t max_groups_buffer_entry_count, RenderInfo *render_info, const CountDistinctDescriptors count_distinct_descriptors, const bool must_use_baseline_sort, const bool output_columnar_hint, const bool streaming_top_n_hint, const bool threads_can_reuse_group_by_buffers)
size_t getEffectiveKeyWidth() const
void codegenApproxQuantile(const size_t target_idx, const Analyzer::Expr *target_expr, std::vector< llvm::Value * > &agg_args, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type)
void checkErrorCode(llvm::Value *retCode)
bool with_dynamic_watchdog
CountDistinctDescriptors init_count_distinct_descriptors(const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &query_infos, const ColRangeInfo &group_by_range_info, const ExecutorDeviceType device_type, Executor *executor)
size_t get_heap_key_slot_index(const std::vector< Analyzer::Expr * > &target_exprs, const size_t target_idx)
const std::list< std::shared_ptr< Analyzer::Expr > > groupby_exprs
bool takes_float_argument(const TargetInfo &target_info)
#define LLVM_ALIGN(alignment)
RUNTIME_EXPORT void agg_mode_func(int64_t *agg, const int64_t val)
bool has_count_distinct(const RelAlgExecutionUnit &ra_exe_unit)
std::tuple< llvm::Value *, llvm::Value * > codegenMultiColumnBaselineHash(const CompilationOptions &co, llvm::Value *groups_buffer, llvm::Value *group_key, llvm::Value *key_size_lv, const QueryMemoryDescriptor &query_mem_desc, const size_t key_width, const int32_t row_size_quad)
CountDistinctImplType impl_type_
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
static WindowFunctionContext * getActiveWindowFunctionContext(Executor *executor)
TargetInfo get_target_info(const Analyzer::Expr *target_expr, const bool bigint_count)
size_t getRowSize() const
Helpers for codegen of target expressions.
size_t getColOnlyOffInBytes(const size_t col_idx) const
size_t get_count_distinct_sub_bitmap_count(const size_t bitmap_sz_bits, const RelAlgExecutionUnit &ra_exe_unit, const ExecutorDeviceType device_type)
int64_t getMaxVal() const
const SQLTypeInfo get_compact_type(const TargetInfo &target)
llvm::Value * codegenWindowPosition(const WindowFunctionContext *window_func_context, llvm::Value *pos_arg)
llvm::LLVMContext & context_
GroupByAndAggregate(Executor *executor, const ExecutorDeviceType device_type, const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &query_infos, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const std::optional< int64_t > &group_cardinality_estimation)
llvm::Value * get_arg_by_name(llvm::Function *func, const std::string &name)
bool useStreamingTopN() const
std::vector< CountDistinctDescriptor > CountDistinctDescriptors
size_t getGroupbyColCount() const
const ColumnDescriptor * get_column_descriptor_maybe(const shared::ColumnKey &column_key)
RUNTIME_EXPORT void agg_count_distinct_skip_val(int64_t *agg, const int64_t val, const int64_t skip_val)
const JoinQualsPerNestingLevel join_quals
llvm::Value * convertNullIfAny(const SQLTypeInfo &arg_type, const TargetInfo &agg_info, llvm::Value *target)
std::optional< size_t > limit
std::list< Analyzer::OrderEntry > order_entries
std::tuple< llvm::Value *, llvm::Value * > codegenSingleColumnPerfectHash(const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, llvm::Value *groups_buffer, llvm::Value *group_expr_lv_translated, llvm::Value *group_expr_lv_original, const int32_t row_size_quad)
bool codegenAggCalls(const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, llvm::Value *varlen_output_buffer, const std::vector< llvm::Value * > &agg_out_vec, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const GpuSharedMemoryContext &gpu_smem_context, DiamondCodegen &diamond_codegen)
std::tuple< llvm::Value *, llvm::Value * > codegenGroupBy(const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, DiamondCodegen &codegen)
void codegen(GroupByAndAggregate *group_by_and_agg, Executor *executor, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const GpuSharedMemoryContext &gpu_smem_context, const std::tuple< llvm::Value *, llvm::Value * > &agg_out_ptr_w_idx, const std::vector< llvm::Value * > &agg_out_vec, llvm::Value *output_buffer_byte_stream, llvm::Value *out_row_idx, llvm::Value *varlen_output_buffer, DiamondCodegen &diamond_codegen) const
int64_t g_bitmap_memory_limit
bool is_distinct_target(const TargetInfo &target_info)
void codegenCountDistinct(const size_t target_idx, const Analyzer::Expr *target_expr, std::vector< llvm::Value * > &agg_args, const QueryMemoryDescriptor &, const ExecutorDeviceType)
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
ExpressionRange getExpressionRange(const Analyzer::BinOper *expr, const std::vector< InputTableInfo > &query_infos, const Executor *, boost::optional< std::list< std::shared_ptr< Analyzer::Expr >>> simple_quals)
const std::shared_ptr< Analyzer::Estimator > estimator
This file includes the class specification for the buffer manager (BufferMgr), and related data struc...
std::tuple< llvm::Value *, llvm::Value * > codegenMultiColumnPerfectHash(llvm::Value *groups_buffer, llvm::Value *group_key, llvm::Value *key_size_lv, const QueryMemoryDescriptor &query_mem_desc, const int32_t row_size_quad)
const SQLTypeInfo & get_type_info() const
QueryDescriptionType getQueryDescriptionType() const
int64_t getShardedTopBucket(const ColRangeInfo &col_range_info, const size_t shard_count) const
ExecutorDeviceType device_type
int64_t getBucket() const
const CountDistinctDescriptor & getCountDistinctDescriptor(const size_t idx) const
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
bool window_function_is_aggregate(const SqlWindowFunctionKind kind)
const std::vector< InputTableInfo > & query_infos_
bool isSingleColumnGroupByWithPerfectHash() const
const shared::ColumnKey & getColumnKey() const
int64_t get_bucketed_cardinality_without_nulls(const ColRangeInfo &col_range_info)
bool expr_is_rowid(const Analyzer::Expr *expr)
std::unique_ptr< QueryMemoryDescriptor > initQueryMemoryDescriptor(const bool allow_multifrag, const size_t max_groups_buffer_entry_count, const int8_t crt_min_byte_width, RenderInfo *render_info, const bool output_columnar_hint)
const ExecutorDeviceType device_type_
void codegenEstimator(std::stack< llvm::BasicBlock * > &array_loops, DiamondCodegen &diamond_codegen, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &)
static size_t shard_count_for_top_groups(const RelAlgExecutionUnit &ra_exe_unit)
std::unordered_map< size_t, SQLTypeInfo > target_exprs_original_type_infos
std::vector< llvm::Value * > codegenAggArg(const Analyzer::Expr *target_expr, const CompilationOptions &co)
llvm::Function * codegenPerfectHashFunction()
llvm::Value * codegenWindowRowPointer(const Analyzer::WindowFunction *window_func, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, DiamondCodegen &diamond_codegen)
bool cardinality_estimate_less_than_column_range(const int64_t cardinality_estimate, const ColRangeInfo &col_range_info)
int32_t get_agg_count(const std::vector< Analyzer::Expr * > &target_exprs)
void add(Value const value)
Descriptor for the result set buffer layout.
const std::optional< int64_t > group_cardinality_estimation_
llvm::Value * codegenOutputSlot(llvm::Value *groups_buffer, const QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, DiamondCodegen &diamond_codegen)
std::list< std::shared_ptr< Analyzer::Expr > > quals
bool didOutputColumnar() const
bool usesGetGroupValueFast() const
bool interleavedBins(const ExecutorDeviceType) const
bool threadsShareMemory() const
int64_t getMinVal() const
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
Estimators to be used when precise cardinality isn't useful.
int64_t get_epoch_days_from_seconds(const int64_t seconds)
RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count(uint64_t *agg, const int64_t)
static size_t getBaselineThreshold(bool for_count_distinct, ExecutorDeviceType device_type)
uint32_t log2_bytes(const uint32_t bytes)
HOST DEVICE bool get_notnull() const
const RelAlgExecutionUnit & ra_exe_unit_
size_t getColOffInBytes(const size_t col_idx) const
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals