33 std::unique_ptr<BoundingBoxIntersectTuningParamRecycler>
35 std::make_unique<BoundingBoxIntersectTuningParamRecycler>();
38 std::shared_ptr<BoundingBoxIntersectJoinHashTable>
40 const std::shared_ptr<Analyzer::BinOper> condition,
41 const std::vector<InputTableInfo>& query_infos,
44 const int device_count,
50 decltype(std::chrono::steady_clock::now()) ts1, ts2;
51 auto copied_query_hints = query_hints;
53 LOG(
INFO) <<
"Ignoring query hint \'force_one_to_many_hash_join\' for bounding box "
55 copied_query_hints.force_one_to_many_hash_join =
false;
58 LOG(
INFO) <<
"Ignoring query hint \'force_baseline_hash_join\' for bounding box "
60 copied_query_hints.force_baseline_hash_join =
false;
62 std::vector<InnerOuter> inner_outer_pairs;
63 if (
const auto range_expr =
64 dynamic_cast<const Analyzer::RangeOper*>(condition->get_right_operand())) {
73 hashtable_build_dag_map,
75 table_id_to_node_map);
81 CHECK(!inner_outer_pairs.empty());
83 const auto getHashTableType =
84 [](
const std::shared_ptr<Analyzer::BinOper> condition,
85 const std::vector<InnerOuter>& inner_outer_pairs) ->
HashType {
87 if (condition->is_bbox_intersect_oper()) {
88 CHECK_EQ(inner_outer_pairs.size(), size_t(1));
89 if (inner_outer_pairs[0].first->get_type_info().is_array() &&
90 inner_outer_pairs[0].second->get_type_info().is_array() &&
92 inner_outer_pairs[0].second->get_type_info().get_size() == 32) {
99 const auto layout = getHashTableType(condition, inner_outer_pairs);
103 <<
" for qual: " << condition->toString();
104 ts1 = std::chrono::steady_clock::now();
107 const auto qi_0 = query_infos[0].info.getNumTuplesUpperBound();
108 const auto qi_1 = query_infos[1].info.getNumTuplesUpperBound();
110 VLOG(1) <<
"table_key = " << query_infos[0].table_key <<
" has " << qi_0 <<
" tuples.";
111 VLOG(1) <<
"table_key = " << query_infos[1].table_key <<
" has " << qi_1 <<
" tuples.";
113 const auto& query_info =
121 auto join_hash_table =
122 std::make_shared<BoundingBoxIntersectJoinHashTable>(condition,
131 hashtable_build_dag_map,
132 table_id_to_node_map);
134 join_hash_table->reify(layout);
136 throw HashJoinFail(std::string(
"Could not build a 1-to-1 correspondence for columns "
137 "involved in bounding box intersection | ") +
141 std::string(
"Could not build hash tables for bounding box intersection | "
142 "Inner table too big. Attempt manual table reordering "
143 "or create a single fragment inner table. | ") +
147 }
catch (
const std::exception& e) {
149 std::string(
"Failed to build hash tables for bounding box intersection | ") +
153 ts2 = std::chrono::steady_clock::now();
155 << std::chrono::duration_cast<std::chrono::milliseconds>(ts2 - ts1).count()
158 return join_hash_table;
164 const std::vector<double>& bucket_sizes,
165 const std::vector<double>& bucket_thresholds,
166 const double initial_value) {
167 std::vector<double> corrected_bucket_sizes(bucket_sizes);
168 for (
size_t i = 0; i != bucket_sizes.size(); ++i) {
169 if (bucket_sizes[i] == initial_value) {
170 corrected_bucket_sizes[i] = bucket_thresholds[i];
173 return corrected_bucket_sizes;
177 const std::vector<double>& bucket_thresholds,
181 const std::vector<InnerOuter>& inner_outer_pairs,
182 const Executor* executor) {
184 CHECK_EQ(inner_outer_pairs.size(), 1u);
186 const auto col = inner_outer_pairs[0].first;
188 const auto col_ti = col->get_type_info();
189 CHECK(col_ti.is_array());
193 const size_t num_dims{2};
194 const double initial_bin_value{0.0};
195 std::vector<double> bucket_sizes(num_dims, initial_bin_value);
196 CHECK_EQ(bucket_thresholds.size(), num_dims);
198 VLOG(1) <<
"Computing x and y bucket sizes for bounding box intersection with maximum "
206 bucket_sizes, join_column, join_column_type, bucket_thresholds, thread_count);
211 const int device_id = 0;
212 auto data_mgr = executor->getDataMgr();
215 auto device_bucket_sizes_gpu =
219 auto device_bucket_thresholds_gpu =
224 join_column_type_gpu,
225 device_bucket_thresholds_gpu);
226 allocator.
copyFromDevice(reinterpret_cast<int8_t*>(bucket_sizes.data()),
227 reinterpret_cast<int8_t*>(device_bucket_sizes_gpu),
228 bucket_sizes.size() *
sizeof(double));
232 bucket_sizes, bucket_thresholds, initial_bin_value);
234 VLOG(1) <<
"Computed x and y bucket sizes for bounding box intersection: ("
235 << corrected_bucket_sizes[0] <<
", " << corrected_bucket_sizes[1] <<
")";
237 return corrected_bucket_sizes;
242 const size_t emitted_keys_count,
243 const size_t hash_table_size,
244 const std::vector<double>& bucket_sizes)
245 : entry_count(entry_count)
246 , emitted_keys_count(emitted_keys_count)
247 , keys_per_bin(entry_count == 0 ? std::numeric_limits<double>::max()
248 : emitted_keys_count / (entry_count / 2.0))
249 , hash_table_size(hash_table_size)
250 , bucket_sizes(bucket_sizes) {}
262 os <<
" entry_count: " << props.
entry_count <<
", emitted_keys "
270 const double bbox_intersect_target_entries_per_bin)
273 , chosen_bbox_intersect_threshold(-1)
275 , crt_reverse_search_iteration(0)
276 , bbox_intersect_max_table_size_bytes(bbox_intersect_max_table_size_bytes)
277 , bbox_intersect_target_entries_per_bin(bbox_intersect_target_entries_per_bin) {}
293 const size_t max_reverse_search_iterations{8};
299 const bool new_bbox_intersect_threshold) {
300 prev_props = crt_props;
301 crt_props = new_props;
304 if (hashTableTooBig() || keysPerBinIncreasing()) {
305 if (hashTableTooBig()) {
306 VLOG(1) <<
"Reached hash table size limit: "
307 << bbox_intersect_max_table_size_bytes <<
" with "
308 << crt_props.hash_table_size <<
" byte hash table, "
309 << crt_props.keys_per_bin <<
" keys per bin.";
310 }
else if (keysPerBinIncreasing()) {
311 VLOG(1) <<
"Keys per bin increasing from " << prev_props.keys_per_bin <<
" to "
312 << crt_props.keys_per_bin;
313 CHECK(previousIterationValid());
315 if (previousIterationValid()) {
316 VLOG(1) <<
"Using previous threshold value " << chosen_bbox_intersect_threshold;
317 crt_props = prev_props;
320 CHECK(hashTableTooBig());
321 crt_reverse_search_iteration++;
322 chosen_bbox_intersect_threshold = new_bbox_intersect_threshold;
324 if (crt_reverse_search_iteration == max_reverse_search_iterations) {
325 VLOG(1) <<
"Hit maximum number (" << max_reverse_search_iterations
326 <<
") of reverse tuning iterations. Aborting tuning";
331 if (crt_reverse_search_iteration > 1 &&
332 crt_props.hash_table_size == prev_props.hash_table_size) {
334 VLOG(1) <<
"Hash table size not decreasing (" << crt_props.hash_table_size
335 <<
" bytes) and still above maximum allowed size ("
336 << bbox_intersect_max_table_size_bytes <<
" bytes). Aborting tuning";
342 if (crt_step == 1 && crt_reverse_search_iteration == 1) {
344 <<
"First iteration of tuning led to hash table size over "
345 "limit. Reversing search to try larger bin sizes (previous threshold: "
346 << chosen_bbox_intersect_threshold <<
")";
348 tuning_direction = TuningDirection::LARGER;
355 chosen_bbox_intersect_threshold = new_bbox_intersect_threshold;
357 if (keysPerBinUnderThreshold()) {
358 VLOG(1) <<
"Hash table reached size " << crt_props.hash_table_size
359 <<
" with keys per bin " << crt_props.keys_per_bin <<
" under threshold "
360 << bbox_intersect_target_entries_per_bin
361 <<
". Terminating bucket size loop.";
365 if (crt_reverse_search_iteration > 0) {
369 VLOG(1) <<
"On reverse (larger tuning direction) search found workable "
370 <<
" hash table size of " << crt_props.hash_table_size
371 <<
" with keys per bin " << crt_props.keys_per_bin
372 <<
". Terminating bucket size loop.";
380 return crt_props.hash_table_size > bbox_intersect_max_table_size_bytes;
384 return crt_props.keys_per_bin > prev_props.keys_per_bin;
388 return tuning_direction == TuningDirection::SMALLER && crt_step > 1;
392 return crt_props.keys_per_bin < bbox_intersect_target_entries_per_bin;
400 const double min_threshold,
402 const std::vector<ColumnsForDevice>& columns_per_device,
403 const std::vector<InnerOuter>& inner_outer_pairs,
404 const size_t table_tuple_count,
405 const Executor* executor)
407 , bucket_thresholds_(num_dims_, bucket_threshold)
409 , min_threshold_(min_threshold)
410 , effective_memory_level_(effective_memory_level)
411 , columns_per_device_(columns_per_device)
412 , inner_outer_pairs_(inner_outer_pairs)
413 , table_tuple_count_(table_tuple_count)
415 CHECK(!columns_per_device_.empty());
421 return tuneOneStep(tuning_direction, step_);
425 const double step_overide) {
426 if (table_tuple_count_ == 0) {
429 if (tuning_direction == TuningState::TuningDirection::SMALLER) {
430 return tuneSmallerOneStep(step_overide);
432 return tuneLargerOneStep(step_overide);
436 return *std::min_element(bucket_thresholds_.begin(), bucket_thresholds_.end());
446 if (num_steps_ == 0) {
447 CHECK_EQ(current_bucket_sizes_.size(),
static_cast<size_t>(0));
448 current_bucket_sizes_ = computeBucketSizes();
450 CHECK_EQ(current_bucket_sizes_.size(), num_dims_);
451 std::vector<double> inverse_bucket_sizes;
452 for (
const auto s : current_bucket_sizes_) {
453 inverse_bucket_sizes.emplace_back(1.0 / s);
455 return inverse_bucket_sizes;
460 for (
const auto& t : bucket_thresholds_) {
461 if (t < min_threshold_) {
469 if (table_tuple_count_ == 0) {
470 return std::vector<double>(num_dims_, 0);
473 effective_memory_level_,
474 columns_per_device_.front().join_columns[0],
475 columns_per_device_.front().join_column_types[0],
481 if (!current_bucket_sizes_.empty()) {
482 CHECK_EQ(current_bucket_sizes_.size(), bucket_thresholds_.size());
483 bucket_thresholds_ = current_bucket_sizes_;
484 for (
auto& t : bucket_thresholds_) {
488 if (bucketThresholdsBelowMinThreshold()) {
489 VLOG(1) <<
"Aborting tuning for bounding box intersection as at least one bucket "
490 "size is below min threshold";
493 const auto next_bucket_sizes = computeBucketSizes();
494 if (next_bucket_sizes == current_bucket_sizes_) {
495 VLOG(1) <<
"Aborting tuning for bounding box intersection as bucket size is no "
500 current_bucket_sizes_ = next_bucket_sizes;
506 if (!current_bucket_sizes_.empty()) {
507 CHECK_EQ(current_bucket_sizes_.size(), bucket_thresholds_.size());
508 bucket_thresholds_ = current_bucket_sizes_;
511 for (
auto& t : bucket_thresholds_) {
521 current_bucket_sizes_ = bucket_thresholds_;
528 size_t num_steps_{0};
543 os <<
"Step Num: " << tuner.
num_steps_ <<
", Threshold: " << std::fixed <<
"("
545 <<
", Step Size: " << std::fixed << tuner.
step_ <<
", Min: " << std::fixed
555 const auto& query_info =
560 <<
", table_id: " << table_id;
561 if (query_info.fragments.empty()) {
566 std::optional<double> bbox_intersect_threshold_override;
568 auto skip_hashtable_caching =
false;
570 VLOG(1) <<
"Setting bounding box intersection bucket threshold "
571 "\'bbox_intersect_bucket_threshold\' via "
577 std::ostringstream oss;
578 oss <<
"User requests to change a threshold \'bbox_intersect_max_table_size_bytes\' "
581 if (!bbox_intersect_threshold_override.has_value()) {
582 oss <<
": " << bbox_intersect_max_table_size_bytes <<
" -> "
586 oss <<
", but is skipped since the query hint also changes the threshold "
587 "\'bbox_intersect_bucket_threshold\'";
589 VLOG(1) << oss.str();
592 VLOG(1) <<
"User requests to skip caching join hashtable for bounding box "
593 "intersection and its tuned "
594 "parameters for this query";
595 skip_hashtable_caching =
true;
598 VLOG(1) <<
"User requests to change a threshold \'bbox_intersect_keys_per_bin\' via "
601 << bbox_intersect_target_entries_per_bin <<
" -> "
611 auto allow_gpu_hashtable_build =
614 if (allow_gpu_hashtable_build) {
615 if (data_mgr->gpusPresent() &&
617 VLOG(1) <<
"A user forces to build GPU hash table for bounding box intersection";
619 allow_gpu_hashtable_build =
false;
620 VLOG(1) <<
"A user forces to build GPU hash table for bounding box intersection "
621 "but we skip it since either GPU is not presented or CPU execution mode "
626 std::vector<ColumnsForDevice> columns_per_device;
627 std::vector<std::unique_ptr<CudaAllocator>> dev_buff_owners;
629 allow_gpu_hashtable_build) {
630 for (
int device_id = 0; device_id <
device_count_; ++device_id) {
631 dev_buff_owners.emplace_back(std::make_unique<CudaAllocator>(
636 std::vector<std::vector<Fragmenter_Namespace::FragmentInfo>> fragments_per_device;
638 size_t total_num_tuples = 0;
639 for (
int device_id = 0; device_id <
device_count_; ++device_id) {
640 fragments_per_device.emplace_back(
643 : query_info.fragments);
644 const size_t crt_num_tuples =
646 fragments_per_device.back().end(),
648 [](
const auto& sum,
const auto& fragment) {
649 return sum + fragment.getNumTuples();
651 total_num_tuples += crt_num_tuples;
652 const auto columns_for_device =
656 allow_gpu_hashtable_build
657 ? dev_buff_owners[device_id].get()
659 columns_per_device.push_back(columns_for_device);
663 auto hashtable_access_path_info =
671 fragments_per_device,
675 table_keys_ = hashtable_access_path_info.table_keys;
677 auto get_inner_table_key = [
this]() {
679 return col_var->getTableKey();
683 const auto& table_key = get_inner_table_key();
689 if (bbox_intersect_threshold_override) {
691 BucketSizeTuner tuner(*bbox_intersect_threshold_override,
699 const auto inverse_bucket_sizes = tuner.getInverseBucketSizes();
701 auto [entry_count, emitted_keys_count] =
703 inverse_bucket_sizes,
705 bbox_intersect_max_table_size_bytes,
706 *bbox_intersect_threshold_override);
712 *bbox_intersect_threshold_override,
713 inverse_bucket_sizes,
714 fragments_per_device,
722 skip_hashtable_caching,
723 bbox_intersect_max_table_size_bytes,
724 *bbox_intersect_threshold_override);
726 double bbox_intersect_bucket_threshold = std::numeric_limits<double>::max();
728 bbox_intersect_bucket_threshold,
730 fragments_per_device,
732 std::vector<size_t> per_device_chunk_key;
734 get_inner_table_key().table_id > 0) {
735 for (
int device_id = 0; device_id <
device_count_; ++device_id) {
740 per_device_chunk_key.push_back(chunk_key_hash);
743 columns_per_device.front().join_columns.front().num_elems,
746 bbox_intersect_max_table_size_bytes,
747 bbox_intersect_bucket_threshold,
756 hashtable_cache_key_.front(),
759 if (cached_bucket_threshold) {
760 bbox_intersect_bucket_threshold = cached_bucket_threshold->bucket_threshold;
761 auto inverse_bucket_sizes = cached_bucket_threshold->bucket_sizes;
763 bbox_intersect_bucket_threshold,
764 inverse_bucket_sizes);
766 bbox_intersect_bucket_threshold,
767 inverse_bucket_sizes,
768 fragments_per_device,
771 if (
auto hash_table =
782 VLOG(1) <<
"Using cached hash table bucket size";
788 hash_table->getEntryCount(),
789 hash_table->getEmittedKeysCount(),
790 skip_hashtable_caching,
791 bbox_intersect_max_table_size_bytes,
792 bbox_intersect_bucket_threshold);
794 VLOG(1) <<
"Computing bucket size for cached bucket threshold";
796 BucketSizeTuner tuner(bbox_intersect_bucket_threshold,
805 const auto inverse_bucket_sizes = tuner.getInverseBucketSizes();
807 auto [entry_count, emitted_keys_count] =
809 inverse_bucket_sizes,
811 bbox_intersect_max_table_size_bytes,
812 bbox_intersect_bucket_threshold);
816 bbox_intersect_bucket_threshold,
817 inverse_bucket_sizes,
818 fragments_per_device,
827 skip_hashtable_caching,
828 bbox_intersect_max_table_size_bytes,
829 bbox_intersect_bucket_threshold);
833 BucketSizeTuner tuner(
834 bbox_intersect_bucket_threshold,
843 VLOG(1) <<
"Running auto tune logic for bounding box intersection with parameters: "
847 TuningState tuning_state(bbox_intersect_max_table_size_bytes,
848 bbox_intersect_target_entries_per_bin);
849 while (tuner.tuneOneStep(tuning_state.tuning_direction)) {
850 const auto inverse_bucket_sizes = tuner.getInverseBucketSizes();
852 const auto [crt_entry_count, crt_emitted_keys_count] =
854 inverse_bucket_sizes,
856 tuning_state.bbox_intersect_max_table_size_bytes,
857 tuning_state.chosen_bbox_intersect_threshold);
859 inverse_bucket_sizes.size(), crt_emitted_keys_count, crt_entry_count);
860 HashTableProps crt_props(crt_entry_count,
861 crt_emitted_keys_count,
863 inverse_bucket_sizes);
864 VLOG(1) <<
"Tuner output: " << tuner <<
" with properties " << crt_props;
866 const auto should_continue = tuning_state(crt_props, tuner.getMinBucketSize());
868 tuning_state.crt_props.bucket_sizes, columns_per_device, device_count_);
869 if (!should_continue) {
874 const auto& crt_props = tuning_state.crt_props;
877 const size_t hash_table_size =
879 crt_props.emitted_keys_count,
880 crt_props.entry_count);
881 CHECK_EQ(crt_props.hash_table_size, hash_table_size);
884 hash_table_size > bbox_intersect_max_table_size_bytes) {
885 VLOG(1) <<
"Could not find suitable parameters to create hash "
886 "table for bounding box intersectionunder max allowed size ("
887 << bbox_intersect_max_table_size_bytes <<
") bytes.";
891 VLOG(1) <<
"Final tuner output: " << tuner <<
" with properties " << crt_props;
893 VLOG(1) <<
"Final bucket sizes: ";
895 VLOG(1) <<
"dim[" << dim
898 CHECK_GE(tuning_state.chosen_bbox_intersect_threshold,
double(0));
900 tuning_state.chosen_bbox_intersect_threshold,
902 fragments_per_device,
904 const auto candidate_auto_tuner_cache_key = hashtable_cache_key_.front();
905 if (skip_hashtable_caching) {
906 VLOG(1) <<
"Skip to add tuned parameters to auto tuner";
909 tuning_state.chosen_bbox_intersect_threshold,
918 bbox_intersect_bucket_threshold = tuning_state.chosen_bbox_intersect_threshold;
923 crt_props.entry_count,
924 crt_props.emitted_keys_count,
925 skip_hashtable_caching,
926 bbox_intersect_max_table_size_bytes,
927 bbox_intersect_bucket_threshold);
933 size_t number_of_dimensions,
934 size_t emitted_keys_count,
935 size_t entry_count)
const {
937 const auto key_component_count = number_of_dimensions;
938 const auto entry_size = key_component_count * key_component_width;
939 const auto keys_for_all_rows = emitted_keys_count;
940 const size_t one_to_many_hash_entries = 2 * entry_count + keys_for_all_rows;
941 const size_t hash_table_size =
942 entry_size * entry_count + one_to_many_hash_entries *
sizeof(int32_t);
943 return hash_table_size;
947 const std::vector<Fragmenter_Namespace::FragmentInfo>& fragments,
952 std::vector<JoinColumn> join_columns;
953 std::vector<std::shared_ptr<Chunk_NS::Chunk>> chunks_owner;
954 std::vector<JoinColumnTypeInfo> join_column_types;
955 std::vector<std::shared_ptr<void>> malloc_owner;
957 const auto inner_col = inner_outer_pair.first;
959 if (inner_cd && inner_cd->isVirtualCol) {
964 effective_memory_level,
971 const auto& ti = inner_col->get_type_info();
975 inline_int_null_value<int64_t>(),
980 <<
"Bounding box intersection currently only supported for arrays.";
982 return {join_columns, join_column_types, chunks_owner, {}, malloc_owner};
986 const size_t shard_count,
987 const std::vector<double>& inverse_bucket_sizes_for_dimension,
988 std::vector<ColumnsForDevice>& columns_per_device,
989 const size_t chosen_max_hashtable_size,
990 const double chosen_bucket_threshold) {
991 CHECK(!inverse_bucket_sizes_for_dimension.empty());
992 const auto [tuple_count, emitted_keys_count] =
995 chosen_max_hashtable_size,
996 chosen_bucket_threshold);
997 const auto entry_count = 2 * std::max(tuple_count,
size_t(1));
999 return std::make_pair(
1001 emitted_keys_count);
1005 const std::vector<double>& inverse_bucket_sizes_for_dimension,
1006 std::vector<ColumnsForDevice>& columns_per_device,
1007 const size_t chosen_max_hashtable_size,
1008 const double chosen_bucket_threshold) {
1019 const auto padded_size_bytes = count_distinct_desc.bitmapPaddedSizeBytes();
1021 CHECK(!columns_per_device.empty() && !columns_per_device.front().join_columns.empty());
1022 if (columns_per_device.front().join_columns.front().num_elems == 0) {
1023 return std::make_pair(0, 0);
1029 for (
size_t device_id = 0; device_id < columns_per_device.size(); ++device_id) {
1030 auto& columns_for_device = columns_per_device[device_id];
1031 columns_for_device.setBucketInfo(inverse_bucket_sizes_for_dimension,
1036 CHECK_EQ(columns_per_device.front().join_columns.size(),
1037 columns_per_device.front().join_buckets.size());
1041 const auto cached_count_info =
1045 if (cached_count_info) {
1046 VLOG(1) <<
"Using a cached tuple count: " << cached_count_info->first
1047 <<
", emitted keys count: " << cached_count_info->second;
1048 return *cached_count_info;
1051 std::vector<uint8_t> hll_buffer_all_cpus(thread_count * padded_size_bytes);
1052 auto hll_result = &hll_buffer_all_cpus[0];
1054 std::vector<int32_t> num_keys_for_row;
1056 num_keys_for_row.resize(columns_per_device.front().join_columns[0].num_elems);
1061 count_distinct_desc.bitmap_sz_bits,
1063 columns_per_device.front().join_columns,
1064 columns_per_device.front().join_column_types,
1065 columns_per_device.front().join_buckets,
1067 for (
int i = 1; i < thread_count; ++i) {
1069 hll_result + i * padded_size_bytes,
1070 size_t(1) << count_distinct_desc.bitmap_sz_bits);
1072 return std::make_pair(
1073 hll_size(hll_result, count_distinct_desc.bitmap_sz_bits),
1074 static_cast<size_t>(num_keys_for_row.size() > 0 ? num_keys_for_row.back() : 0));
1077 auto data_mgr =
executor_->getDataMgr();
1078 std::vector<std::vector<uint8_t>> host_hll_buffers(
device_count_);
1079 for (
auto& host_hll_buffer : host_hll_buffers) {
1080 host_hll_buffer.resize(count_distinct_desc.bitmapPaddedSizeBytes());
1082 std::vector<size_t> emitted_keys_count_device_threads(
device_count_, 0);
1083 std::vector<std::future<void>> approximate_distinct_device_threads;
1084 for (
int device_id = 0; device_id <
device_count_; ++device_id) {
1085 approximate_distinct_device_threads.emplace_back(
std::async(
1088 &columns_per_device,
1089 &count_distinct_desc,
1092 &emitted_keys_count_device_threads] {
1093 auto allocator = std::make_unique<CudaAllocator>(
1095 auto device_hll_buffer =
1096 allocator->alloc(count_distinct_desc.bitmapPaddedSizeBytes());
1097 data_mgr->getCudaMgr()->zeroDeviceMem(
1099 count_distinct_desc.bitmapPaddedSizeBytes(),
1102 const auto& columns_for_device = columns_per_device[device_id];
1104 columns_for_device.join_columns, *allocator);
1106 CHECK_GT(columns_for_device.join_buckets.size(), 0u);
1107 const auto& inverse_bucket_sizes_for_dimension =
1108 columns_for_device.join_buckets[0].inverse_bucket_sizes_for_dimension;
1109 auto inverse_bucket_sizes_gpu = allocator->alloc(
1110 inverse_bucket_sizes_for_dimension.size() *
sizeof(double));
1111 allocator->copyToDevice(
1112 inverse_bucket_sizes_gpu,
1113 inverse_bucket_sizes_for_dimension.data(),
1114 inverse_bucket_sizes_for_dimension.size() *
sizeof(double));
1115 const size_t row_counts_buffer_sz =
1116 columns_per_device.front().join_columns[0].num_elems *
sizeof(int32_t);
1117 auto row_counts_buffer = allocator->alloc(row_counts_buffer_sz);
1118 data_mgr->getCudaMgr()->zeroDeviceMem(
1120 row_counts_buffer_sz,
1124 inverse_bucket_sizes_for_dimension.size(),
1126 reinterpret_cast<double*
>(inverse_bucket_sizes_gpu));
1127 const auto key_handler_gpu =
1130 reinterpret_cast<uint8_t*>(device_hll_buffer),
1131 count_distinct_desc.bitmap_sz_bits,
1132 reinterpret_cast<int32_t*>(row_counts_buffer),
1134 columns_for_device.join_columns[0].num_elems);
1136 auto& host_emitted_keys_count = emitted_keys_count_device_threads[device_id];
1137 allocator->copyFromDevice(
1138 &host_emitted_keys_count,
1140 (columns_per_device.front().join_columns[0].num_elems - 1) *
1144 auto& host_hll_buffer = host_hll_buffers[device_id];
1145 allocator->copyFromDevice(&host_hll_buffer[0],
1147 count_distinct_desc.bitmapPaddedSizeBytes());
1150 for (
auto& child : approximate_distinct_device_threads) {
1154 auto& result_hll_buffer = host_hll_buffers.front();
1155 auto hll_result =
reinterpret_cast<int32_t*
>(&result_hll_buffer[0]);
1156 for (
int device_id = 1; device_id <
device_count_; ++device_id) {
1157 auto& host_hll_buffer = host_hll_buffers[device_id];
1159 reinterpret_cast<int32_t*>(&host_hll_buffer[0]),
1160 size_t(1) << count_distinct_desc.bitmap_sz_bits);
1162 const size_t emitted_keys_count =
1164 emitted_keys_count_device_threads.end(),
1166 return std::make_pair(
hll_size(hll_result, count_distinct_desc.bitmap_sz_bits),
1167 emitted_keys_count);
1175 const std::vector<double>& inverse_bucket_sizes,
1176 std::vector<ColumnsForDevice>& columns_per_device,
1177 const size_t device_count) {
1182 CHECK_EQ(columns_per_device.size(),
static_cast<size_t>(device_count));
1183 for (
size_t device_id = 0; device_id < device_count; ++device_id) {
1184 auto& columns_for_device = columns_per_device[device_id];
1219 }
catch (
const std::exception& e) {
1220 VLOG(1) <<
"Caught exception while building baseline hash table for bounding box "
1228 std::vector<ColumnsForDevice>& columns_per_device,
1231 const size_t shard_count,
1232 const size_t entry_count,
1233 const size_t emitted_keys_count,
1234 const bool skip_hashtable_caching,
1235 const size_t chosen_max_hashtable_size,
1236 const double chosen_bucket_threshold) {
1237 std::vector<std::future<void>> init_threads;
1244 for (
int device_id = 0; device_id <
device_count_; ++device_id) {
1245 const auto fragments =
1252 columns_per_device[device_id],
1256 skip_hashtable_caching,
1260 for (
auto& init_thread : init_threads) {
1263 for (
auto& init_thread : init_threads) {
1271 const size_t entry_count,
1272 const size_t emitted_keys_count,
1273 const bool skip_hashtable_caching,
1274 const int device_id,
1289 VLOG(1) <<
"Building join hash table for bounding box intersection on CPU.";
1293 hash_table_entry_info,
1294 skip_hashtable_caching);
1299 auto gpu_hash_table = copyCpuHashTableToGpu(hash_table, device_id);
1313 auto hash_table = initHashTableOnGpu(columns_for_device.
join_columns,
1316 hash_table_entry_info,
1327 const std::vector<JoinColumn>& join_columns,
1328 const std::vector<JoinColumnTypeInfo>& join_column_types,
1329 const std::vector<JoinBucketInfo>& join_bucket_info,
1331 const bool skip_hashtable_caching) {
1333 decltype(std::chrono::steady_clock::now()) ts1, ts2;
1334 ts1 = std::chrono::steady_clock::now();
1335 CHECK(!join_columns.empty());
1336 CHECK(!join_bucket_info.empty());
1339 if (
auto generic_hash_table =
1343 if (
auto hash_table =
1344 std::dynamic_pointer_cast<BaselineHashTable>(generic_hash_table)) {
1345 VLOG(1) <<
"Using cached CPU hash table for initialization.";
1354 if (hash_table_layout == hash_table->getLayout()) {
1360 const auto key_component_count =
1361 join_bucket_info[0].inverse_bucket_sizes_for_dimension.size();
1364 key_component_count,
1366 join_bucket_info[0].inverse_bucket_sizes_for_dimension.data());
1369 dummy_str_proxy_translation_maps_ptrs_and_offsets;
1376 dummy_str_proxy_translation_maps_ptrs_and_offsets,
1377 hash_table_entry_info,
1381 ts2 = std::chrono::steady_clock::now();
1383 throw HashJoinFail(std::string(
"Unrecognized error when initializing CPU hash table "
1384 "for bounding box intersection(") +
1387 std::shared_ptr<BaselineHashTable> hash_table = builder.getHashTable();
1388 if (skip_hashtable_caching) {
1389 VLOG(1) <<
"Skip to cache join hashtable for bounding box intersection";
1391 auto hashtable_build_time =
1392 std::chrono::duration_cast<std::chrono::milliseconds>(ts2 - ts1).count();
1397 hashtable_build_time);
1404 std::shared_ptr<BaselineHashTable> BoundingBoxIntersectJoinHashTable::initHashTableOnGpu(
1405 const std::vector<JoinColumn>& join_columns,
1406 const std::vector<JoinColumnTypeInfo>& join_column_types,
1407 const std::vector<JoinBucketInfo>& join_bucket_info,
1409 const size_t device_id) {
1412 VLOG(1) <<
"Building join hash table for bounding box intersection on GPU.";
1415 auto data_mgr =
executor_->getDataMgr();
1420 CHECK(!join_bucket_info.empty());
1421 auto& inverse_bucket_sizes_for_dimension =
1422 join_bucket_info[0].inverse_bucket_sizes_for_dimension;
1424 inverse_bucket_sizes_for_dimension, allocator);
1425 const auto key_handler =
1428 inverse_bucket_sizes_gpu);
1433 hash_table_entry_info,
1438 throw HashJoinFail(std::string(
"Unrecognized error when initializing GPU hash table "
1439 "for bounding box intersection (") +
1445 std::shared_ptr<BaselineHashTable>
1446 BoundingBoxIntersectJoinHashTable::copyCpuHashTableToGpu(
1447 std::shared_ptr<BaselineHashTable>& cpu_hash_table,
1448 const size_t device_id) {
1451 auto data_mgr =
executor_->getDataMgr();
1457 std::shared_ptr<BaselineHashTable> gpu_hash_table = gpu_builder.
getHashTable();
1458 CHECK(gpu_hash_table);
1459 auto gpu_buffer_ptr = gpu_hash_table->getGpuBuffer();
1460 CHECK(gpu_buffer_ptr);
1464 auto device_allocator = std::make_unique<CudaAllocator>(
1466 device_allocator->copyToDevice(
1468 cpu_hash_table->getCpuBuffer(),
1470 return gpu_hash_table;
1475 #define LL_CONTEXT executor_->cgen_state_->context_
1476 #define LL_BUILDER executor_->cgen_state_->ir_builder_
1477 #define LL_INT(v) executor_->cgen_state_->llInt(v)
1478 #define LL_FP(v) executor_->cgen_state_->llFp(v)
1479 #define ROW_FUNC executor_->cgen_state_->row_func_
1484 CHECK(key_component_width == 4 || key_component_width == 8);
1486 llvm::Value* key_buff_lv{
nullptr};
1487 switch (key_component_width) {
1501 const auto outer_geo = inner_outer_pair.second;
1502 const auto outer_geo_ti = outer_geo->get_type_info();
1504 llvm::Value* arr_ptr =
nullptr;
1508 if (outer_geo_ti.is_geometry()) {
1513 if (
const auto outer_geo_col = dynamic_cast<const Analyzer::ColumnVar*>(outer_geo)) {
1514 const auto outer_geo_col_lvs = code_generator.
codegen(outer_geo_col,
true, co);
1515 CHECK_EQ(outer_geo_col_lvs.size(), size_t(1));
1516 auto column_key = outer_geo_col->getColumnKey();
1517 column_key.column_id = column_key.column_id + 1;
1521 const auto array_ptr = executor_->cgen_state_->emitExternalCall(
1523 llvm::Type::getInt8PtrTy(executor_->cgen_state_->context_),
1524 {outer_geo_col_lvs.front(), code_generator.
posArg(outer_geo_col)});
1525 CHECK(coords_cd->columnType.get_elem_type().get_type() ==
kTINYINT)
1526 <<
"Bounding box intersection only supports TINYINT coordinates columns.";
1528 coords_cd->columnType.get_elem_type());
1529 }
else if (
const auto outer_geo_function_operator =
1530 dynamic_cast<const Analyzer::GeoOperator*>(outer_geo)) {
1532 const auto outer_geo_function_operator_lvs =
1533 code_generator.
codegen(outer_geo_function_operator,
true, co);
1534 CHECK_EQ(outer_geo_function_operator_lvs.size(), size_t(2));
1535 arr_ptr = outer_geo_function_operator_lvs.front();
1536 }
else if (
const auto outer_geo_expr =
1537 dynamic_cast<const Analyzer::GeoExpr*>(outer_geo)) {
1540 }
else if (outer_geo_ti.is_fixlen_array()) {
1542 const auto outer_geo_cast_coord_array =
1546 outer_geo_cast_coord_array->get_operand());
1547 CHECK(outer_geo_coord_array);
1548 CHECK(outer_geo_coord_array->isLocalAlloc());
1549 CHECK_EQ(outer_geo_coord_array->getElementCount(), 2);
1553 CHECK_EQ(outer_geo_ti.get_size(), int(2 * elem_size));
1554 const auto outer_geo_constructed_lvs = code_generator.codegen(outer_geo,
true, co);
1556 const auto array_ptr = outer_geo_constructed_lvs.front();
1558 array_ptr->getType()->getScalarType()->getPointerElementType(),
1565 <<
"Bounding box intersection currently only supports geospatial columns and "
1566 "constructed points.";
1569 for (
size_t i = 0; i < 2; i++) {
1570 const auto key_comp_dest_lv =
LL_BUILDER.CreateGEP(
1571 key_buff_lv->getType()->getScalarType()->getPointerElementType(),
1579 ? executor_->cgen_state_->emitExternalCall(
1580 "get_bucket_key_for_range_compressed",
1582 {arr_ptr,
LL_INT(i),
LL_FP(inverse_bucket_sizes_for_dimension_[i])})
1583 : executor_->cgen_state_->emitExternalCall(
1584 "get_bucket_key_for_range_double",
1586 {arr_ptr,
LL_INT(i),
LL_FP(inverse_bucket_sizes_for_dimension_[i])});
1589 LL_BUILDER.CreateStore(col_lv, key_comp_dest_lv);
1598 CHECK(key_component_width == 4 || key_component_width == 8);
1603 VLOG(1) <<
"Performing codgen for ManyToMany";
1605 const auto outer_col = inner_outer_pair.second;
1608 const auto col_lvs = code_generator.
codegen(outer_col,
true, co);
1609 CHECK_EQ(col_lvs.size(), size_t(1));
1612 CHECK(outer_col_var);
1613 const auto coords_cd =
1617 const auto array_ptr = executor_->cgen_state_->emitExternalCall(
1619 llvm::Type::getInt8PtrTy(executor_->cgen_state_->context_),
1620 {col_lvs.front(), code_generator.
posArg(outer_col)});
1626 array_ptr->setName(
"array_ptr");
1628 auto num_keys_lv = executor_->cgen_state_->emitExternalCall(
1629 "get_num_buckets_for_bounds",
1633 LL_FP(inverse_bucket_sizes_for_dimension_[0]),
1634 LL_FP(inverse_bucket_sizes_for_dimension_[1])});
1635 num_keys_lv->setName(
"num_keys_lv");
1637 return {num_keys_lv, array_ptr};
1642 const size_t index) {
1645 VLOG(1) <<
"Building codegenMatchingSet for ManyToMany";
1647 CHECK(key_component_width == 4 || key_component_width == 8);
1650 const auto composite_dict_ptr_type =
1651 llvm::Type::getIntNPtrTy(
LL_CONTEXT, key_component_width * 8);
1652 const auto composite_key_dict =
1653 hash_ptr->getType()->isPointerTy()
1654 ?
LL_BUILDER.CreatePointerCast(hash_ptr, composite_dict_ptr_type)
1655 :
LL_BUILDER.CreateIntToPtr(hash_ptr, composite_dict_ptr_type);
1658 auto one_to_many_ptr = hash_ptr;
1660 if (one_to_many_ptr->getType()->isPointerTy()) {
1664 CHECK(one_to_many_ptr->getType()->isIntegerTy(64));
1674 const int max_array_size = 200;
1676 const auto out_arr_lv =
LL_BUILDER.CreateAlloca(arr_type);
1677 out_arr_lv->setName(
"out_arr");
1679 const auto casted_out_arr_lv =
1680 LL_BUILDER.CreatePointerCast(out_arr_lv, arr_type->getPointerTo());
1682 const auto element_ptr =
LL_BUILDER.CreateGEP(arr_type, casted_out_arr_lv,
LL_INT(0));
1684 auto rowid_ptr_i32 =
1687 const auto candidate_count_lv = executor_->cgen_state_->emitExternalCall(
1688 "get_candidate_rows",
1693 many_to_many_args[1],
1697 many_to_many_args[0],
1698 LL_INT(key_component_count),
1701 LL_INT(composite_key_dict_size),
1705 const auto slot_lv =
LL_INT(int64_t(0));
1707 return {rowid_ptr_i32, candidate_count_lv, slot_lv};
1709 VLOG(1) <<
"Building codegenMatchingSet for Baseline";
1712 CHECK(key_component_width == 4 || key_component_width == 8);
1716 const auto composite_dict_ptr_type =
1717 llvm::Type::getIntNPtrTy(
LL_CONTEXT, key_component_width * 8);
1718 const auto composite_key_dict =
1719 hash_ptr->getType()->isPointerTy()
1720 ?
LL_BUILDER.CreatePointerCast(hash_ptr, composite_dict_ptr_type)
1721 :
LL_BUILDER.CreateIntToPtr(hash_ptr, composite_dict_ptr_type);
1723 const auto key = executor_->cgen_state_->emitExternalCall(
1724 "get_composite_key_index_" +
std::to_string(key_component_width * 8),
1727 LL_INT(key_component_count),
1730 auto one_to_many_ptr = hash_ptr;
1731 if (one_to_many_ptr->getType()->isPointerTy()) {
1735 CHECK(one_to_many_ptr->getType()->isIntegerTy(64));
1741 std::vector<llvm::Value*>{
1755 const int device_id,
1764 auto buffer_size = hash_table->getHashTableBufferSize(device_type);
1766 std::unique_ptr<int8_t[]> buffer_copy;
1768 buffer_copy = std::make_unique<int8_t[]>(buffer_size);
1770 auto data_mgr = executor_->getDataMgr();
1771 auto device_allocator = std::make_unique<CudaAllocator>(
1774 device_allocator->copyFromDevice(buffer_copy.get(), buffer, buffer_size);
1776 auto ptr1 = buffer_copy ? buffer_copy.get() :
reinterpret_cast<const int8_t*
>(buffer);
1778 auto ptr1 =
reinterpret_cast<const int8_t*
>(buffer);
1790 hash_table->getEntryCount(),
1801 const int device_id)
const {
1805 auto buffer_size = hash_table->getHashTableBufferSize(device_type);
1807 std::unique_ptr<int8_t[]> buffer_copy;
1809 buffer_copy = std::make_unique<int8_t[]>(buffer_size);
1811 auto data_mgr = executor_->getDataMgr();
1812 auto allocator = std::make_unique<CudaAllocator>(
1815 allocator->copyFromDevice(buffer_copy.get(), buffer, buffer_size);
1817 auto ptr1 = buffer_copy ? buffer_copy.get() :
reinterpret_cast<const int8_t*
>(buffer);
1819 auto ptr1 =
reinterpret_cast<const int8_t*
>(buffer);
1827 hash_table->getEntryCount(),
1836 const std::vector<InnerOuter>& inner_outer_pairs)
const {
1839 this->executor_->getDataMgr()->gpusPresent() &&
1861 VLOG(1) <<
"Checking CPU hash table cache.";
1865 auto cached_hashtable =
1866 hash_table_cache_->getItemFromCache(key, item_type, device_identifier, meta_info);
1867 if (cached_hashtable) {
1868 return cached_hashtable;
1873 std::optional<std::pair<size_t, size_t>>
1881 auto cached_hashtable =
1882 hash_table_cache_->getItemFromCache(key, item_type, device_identifier, metaInfo);
1883 if (cached_hashtable) {
1884 return std::make_pair(cached_hashtable->getEntryCount() / 2,
1885 cached_hashtable->getEmittedKeysCount());
1887 return std::nullopt;
1893 std::shared_ptr<HashTable> hashtable_ptr,
1895 size_t hashtable_building_time) {
1897 CHECK(hashtable_ptr && !hashtable_ptr->getGpuBuffer());
1907 hashtable_building_time,
static std::vector< int > collectFragmentIds(const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments)
bool previousIterationValid() const
std::vector< double > compute_bucket_sizes(const std::vector< double > &bucket_thresholds, const Data_Namespace::MemoryLevel effective_memory_level, const JoinColumn &join_column, const JoinColumnTypeInfo &join_column_type, const std::vector< InnerOuter > &inner_outer_pairs, const Executor *executor)
static std::unique_ptr< BoundingBoxIntersectTuningParamRecycler > auto_tuner_cache_
size_t getKeyComponentCount() const
bool tuneOneStep(const TuningState::TuningDirection tuning_direction, const double step_overide)
virtual HashJoinMatchingSet codegenMatchingSet(const CompilationOptions &, const size_t)=0
bool hashTableTooBig() const
void setBoundingBoxIntersectionMetaInfo(size_t max_table_size_bytes, double bucket_threshold, std::vector< double > &bucket_sizes)
virtual std::pair< size_t, size_t > computeHashTableCounts(const size_t shard_count, const std::vector< double > &inverse_bucket_sizes_for_dimension, std::vector< ColumnsForDevice > &columns_per_device, const size_t chosen_max_hashtable_size, const double chosen_bucket_threshold)
HashtableCacheMetaInfo hashtable_cache_meta_info_
std::vector< double > computeBucketSizes() const
auto getMinBucketSize() const
std::mutex cpu_hash_table_buff_mutex_
static llvm::Value * codegenHashTableLoad(const size_t table_idx, Executor *executor)
Data_Namespace::MemoryLevel getEffectiveMemoryLevel(const std::vector< InnerOuter > &inner_outer_pairs) const
size_t calculateHashTableSize(size_t number_of_dimensions, size_t emitted_keys_count, size_t entry_count) const
static bool isInvalidHashTableCacheKey(const std::vector< QueryPlanHash > &cache_keys)
std::vector< double > bucket_thresholds_
llvm::Value * codegenKey(const CompilationOptions &)
size_t getKeyComponentWidth() const
shared::TableKey getInnerTableId() const noexceptoverride
const std::vector< ColumnsForDevice > & columns_per_device_
std::vector< ChunkKey > cache_key_chunks
std::vector< QueryPlanHash > hashtable_cache_key_
T * transfer_flat_object_to_gpu(const T &object, DeviceAllocator &allocator)
HashJoinMatchingSet codegenMatchingSet(const CompilationOptions &, const size_t) override
std::unordered_set< size_t > table_keys_
double bbox_intersect_keys_per_bin
std::ostream & operator<<(std::ostream &os, const SessionInfo &session_info)
size_t offsetBufferOff() const noexceptoverride
std::vector< double > current_bucket_sizes_
void hll_unify(T1 *lhs, T2 *rhs, const size_t m)
JoinColumn fetchJoinColumn(const Analyzer::ColumnVar *hash_col, const std::vector< Fragmenter_Namespace::FragmentInfo > &fragment_info, const Data_Namespace::MemoryLevel effective_memory_level, const int device_id, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, DeviceAllocator *dev_buff_owner, std::vector< std::shared_ptr< void >> &malloc_owner, Executor *executor, ColumnCacheMap *column_cache)
void reify(const HashType preferred_layout)
bool bucketThresholdsBelowMinThreshold() const
llvm::Value * posArg(const Analyzer::Expr *) const
std::vector< std::shared_ptr< HashTable > > hash_tables_for_device_
const std::shared_ptr< Analyzer::BinOper > condition_
llvm::Value * castArrayPointer(llvm::Value *ptr, const SQLTypeInfo &elem_ti)
std::shared_ptr< BaselineHashTable > initHashTableOnCpu(const std::vector< JoinColumn > &join_columns, const std::vector< JoinColumnTypeInfo > &join_column_types, const std::vector< JoinBucketInfo > &join_bucket_info, const BaselineHashTableEntryInfo hash_table_entry_info, const bool skip_hashtable_caching)
const ColumnDescriptor * get_metadata_for_column(const ::shared::ColumnKey &column_key)
const InputTableInfo & get_inner_query_info(const shared::TableKey &inner_table_key, const std::vector< InputTableInfo > &query_infos)
std::vector< double > inverse_bucket_sizes_for_dimension_
double g_bbox_intersect_target_entries_per_bin
#define DEBUG_TIMER_NEW_THREAD(parent_thread_id)
DecodedJoinHashBufferSet toSet(const ExecutorDeviceType device_type, const int device_id) const override
std::vector< FragmentInfo > fragments
CompositeKeyInfo composite_key_info_
std::optional< HashType > layout_override_
virtual std::pair< size_t, size_t > approximateTupleCount(const std::vector< double > &inverse_bucket_sizes_for_dimension, std::vector< ColumnsForDevice > &, const size_t chosen_max_hashtable_size, const double chosen_bucket_threshold)
void allocateDeviceMemory(const BaselineHashTableEntryInfo hash_table_entry_info, const int device_id, const Executor *executor, const RegisteredQueryHint &query_hint)
double chosen_bbox_intersect_bucket_threshold_
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
std::vector< llvm::Value * > codegenManyKey(const CompilationOptions &)
size_t hll_size(const T *M, const size_t bitmap_sz_bits)
int initHashTableOnGpu(KEY_HANDLER *key_handler, const std::vector< JoinColumn > &join_columns, const JoinType join_type, const BaselineHashTableEntryInfo hash_table_entry_info, const int device_id, const Executor *executor, const RegisteredQueryHint &query_hint)
size_t shardCount() const
void setInverseBucketSizeInfo(const std::vector< double > &inverse_bucket_sizes, std::vector< ColumnsForDevice > &columns_per_device, const size_t device_count)
HashTableBuildDagMap hashtable_build_dag_map_
size_t crt_reverse_search_iteration
const double min_threshold_
const std::vector< JoinColumnTypeInfo > join_column_types
void compute_bucket_sizes_on_device(double *bucket_sizes_buffer, const JoinColumn *join_column, const JoinColumnTypeInfo *type_info, const double *bucket_size_thresholds)
bool isBitwiseEq() const override
ColumnsForDevice fetchColumnsForDevice(const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const int device_id, DeviceAllocator *dev_buff_owner)
bool keysPerBinIncreasing() const
future< Result > async(Fn &&fn, Args &&...args)
std::optional< BoundingBoxIntersectMetaInfo > getBoundingBoxIntersectMetaInfo()
std::unordered_map< size_t, HashTableBuildDag > HashTableBuildDagMap
std::vector< double > correct_uninitialized_bucket_sizes_to_thresholds(const std::vector< double > &bucket_sizes, const std::vector< double > &bucket_thresholds, const double initial_value)
const Executor * executor_
HashType getHashTableLayout() const
size_t bbox_intersect_max_table_size_bytes
const ColumnDescriptor * get_column_descriptor_maybe(const shared::ColumnKey &column_key)
std::vector< double > getInverseBucketSizes()
void compute_bucket_sizes_on_cpu(std::vector< double > &bucket_sizes_for_dimension, const JoinColumn &join_column, const JoinColumnTypeInfo &type_info, const std::vector< double > &bucket_size_thresholds, const int thread_count)
const Data_Namespace::MemoryLevel memory_level_
double bbox_intersect_bucket_threshold
static std::unique_ptr< HashtableRecycler > hash_table_cache_
void generateCacheKey(const size_t max_hashtable_size, const double bucket_threshold, const std::vector< double > &bucket_sizes, std::vector< std::vector< Fragmenter_Namespace::FragmentInfo >> &fragments_per_device, int device_count)
std::vector< Fragmenter_Namespace::FragmentInfo > only_shards_for_device(const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const int device_id, const int device_count)
static std::shared_ptr< BoundingBoxIntersectJoinHashTable > getInstance(const std::shared_ptr< Analyzer::BinOper > condition, const std::vector< InputTableInfo > &query_infos, const Data_Namespace::MemoryLevel memory_level, const JoinType join_type, const int device_count, ColumnCacheMap &column_cache, Executor *executor, const HashTableBuildDagMap &hashtable_build_dag_map, const RegisteredQueryHint &query_hint, const TableIdToNodeMap &table_id_to_node_map)
Make hash table from an in-flight SQL query's parse tree etc.
const std::vector< InnerOuter > & inner_outer_pairs_
double chosen_bbox_intersect_threshold
const std::vector< InputTableInfo > & query_infos_
static constexpr size_t MAX_NUM_HASH_ENTRIES
int8_t * getJoinHashBuffer(const ExecutorDeviceType device_type, const int device_id) const
DEVICE auto accumulate(ARGS &&...args)
virtual void reifyImpl(std::vector< ColumnsForDevice > &columns_per_device, const Fragmenter_Namespace::TableInfo &query_info, const HashType layout, const size_t shard_count, const size_t entry_count, const size_t emitted_keys_count, const bool skip_hashtable_caching, const size_t chosen_max_hashtable_size, const double chosen_bucket_threshold)
BucketSizeTuner(const double bucket_threshold, const double step, const double min_threshold, const Data_Namespace::MemoryLevel effective_memory_level, const std::vector< ColumnsForDevice > &columns_per_device, const std::vector< InnerOuter > &inner_outer_pairs, const size_t table_tuple_count, const Executor *executor)
static std::unordered_set< size_t > getAlternativeTableKeys(const std::vector< ChunkKey > &chunk_keys, const shared::TableKey &inner_table_key)
QueryPlanHash getAlternativeCacheKey(AlternativeCacheKeyForBoundingBoxIntersection &info)
size_t payloadBufferOff() const noexceptoverride
HashTableProps prev_props
void approximate_distinct_tuples_on_device_bbox_intersect(uint8_t *hll_buffer, const uint32_t b, int32_t *row_counts_buffer, const BoundingBoxIntersectKeyHandler *key_handler, const int64_t num_elems)
static std::shared_ptr< RangeJoinHashTable > getInstance(const std::shared_ptr< Analyzer::BinOper > condition, const Analyzer::RangeOper *range_expr, const std::vector< InputTableInfo > &query_infos, const Data_Namespace::MemoryLevel memory_level, const JoinType join_type, const int device_count, ColumnCacheMap &column_cache, Executor *executor, const HashTableBuildDagMap &hashtable_build_dag_map, const RegisteredQueryHint &query_hints, const TableIdToNodeMap &table_id_to_node_map)
TuningState(const size_t bbox_intersect_max_table_size_bytes, const double bbox_intersect_target_entries_per_bin)
HashTable * getHashTableForDevice(const size_t device_id) const
virtual shared::TableKey getInnerTableId() const noexcept=0
std::unordered_map< shared::TableKey, const RelAlgNode * > TableIdToNodeMap
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
HashType getHashType() const noexceptoverride
std::pair< std::vector< const int32_t * >, std::vector< int32_t >> StrProxyTranslationMapsPtrsAndOffsets
std::vector< double > bucket_sizes
std::unique_ptr< BaselineHashTable > getHashTable()
static std::string getHashTypeString(HashType ht) noexcept
std::vector< InnerOuter > inner_outer_pairs_
size_t chosen_bbox_intersect_max_table_size_bytes_
size_t getEntryCount() const
static std::string toString(const std::string &type, const std::string &layout_type, size_t key_component_count, size_t key_component_width, size_t entry_count, const int8_t *ptr1, const int8_t *ptr2, const int8_t *ptr3, const int8_t *ptr4, size_t buffer_size, bool raw=false)
Decode hash table into a human-readable string.
LocalIdsScopeGuard setNewThreadId() const
size_t get_entries_per_device(const size_t total_entries, const size_t shard_count, const size_t device_count, const Data_Namespace::MemoryLevel memory_level)
bool isHintRegistered(const QueryHint hint) const
int initHashTableOnCpu(KEY_HANDLER *key_handler, const CompositeKeyInfo &composite_key_info, const std::vector< JoinColumn > &join_columns, const std::vector< JoinColumnTypeInfo > &join_column_types, const std::vector< JoinBucketInfo > &join_bucket_info, const StrProxyTranslationMapsPtrsAndOffsets &str_proxy_translation_maps_ptrs_and_offsets, const BaselineHashTableEntryInfo hash_table_entry_info, const JoinType join_type, const Executor *executor, const RegisteredQueryHint &query_hint)
bool operator()(const HashTableProps &new_props, const bool new_bbox_intersect_threshold)
virtual void reifyWithLayout(const HashType layout)
std::unordered_map< shared::TableKey, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
bool tuneLargerOneStep(const double step_overide)
const JoinType join_type_
void approximate_distinct_tuples_bbox_intersect(uint8_t *hll_buffer_all_cpus, std::vector< int32_t > &row_counts, const uint32_t b, const size_t padded_size_bytes, const std::vector< JoinColumn > &join_column_per_key, const std::vector< JoinColumnTypeInfo > &type_info_per_key, const std::vector< JoinBucketInfo > &join_buckets_per_key, const int thread_count)
bool keysPerBinUnderThreshold() const
size_t bbox_intersect_max_size
CUstream getQueryEngineCudaStreamForDevice(int device_num)
bool force_baseline_hash_join
const Data_Namespace::MemoryLevel effective_memory_level_
std::size_t hash_value(RexAbstractInput const &rex_ab_input)
size_t getNumTuplesUpperBound() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
RegisteredQueryHint query_hints_
void putHashTableOnCpuToCache(QueryPlanHash key, CacheItemType item_type, std::shared_ptr< HashTable > hashtable_ptr, DeviceIdentifier device_identifier, size_t hashtable_building_time)
#define DEBUG_TIMER(name)
static std::pair< std::vector< InnerOuter >, std::vector< InnerOuterStringOpInfos > > normalizeColumnPairs(const Analyzer::BinOper *condition, const TemporaryTables *temporary_tables)
void reifyForDevice(const ColumnsForDevice &columns_for_device, const HashType layout, const size_t entry_count, const size_t emitted_keys_count, const bool skip_hashtable_caching, const int device_id, const logger::ThreadLocalIds parent_thread_local_ids)
double bbox_intersect_target_entries_per_bin
size_t countBufferOff() const noexceptoverride
bool tuneOneStep(const TuningState::TuningDirection tuning_direction)
bool bbox_intersect_allow_gpu_build
ColumnCacheMap & column_cache_
std::optional< std::pair< size_t, size_t > > getApproximateTupleCountFromCache(QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier)
static DecodedJoinHashBufferSet toSet(size_t key_component_count, size_t key_component_width, size_t entry_count, const int8_t *ptr1, const int8_t *ptr2, const int8_t *ptr3, const int8_t *ptr4, size_t buffer_size)
Decode hash table into a std::set for easy inspection and validation.
T * transfer_vector_of_flat_objects_to_gpu(const std::vector< T > &vec, DeviceAllocator &allocator)
std::vector< JoinBucketInfo > join_buckets
static HashTableProps invalid()
std::shared_ptr< HashTable > initHashTableOnCpuFromCache(QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier)
void copyFromDevice(void *host_dst, const void *device_src, const size_t num_bytes) const override
static constexpr DeviceIdentifier CPU_DEVICE_IDENTIFIER
llvm::ArrayType * get_int_array_type(int const width, int count, llvm::LLVMContext &context)
std::string toString(const ExecutorDeviceType device_type, const int device_id=0, bool raw=false) const override
static HashtableAccessPathInfo getHashtableAccessPathInfo(const std::vector< InnerOuter > &inner_outer_pairs, const std::vector< InnerOuterStringOpInfos > &inner_outer_string_op_infos_pairs, const SQLOps op_type, const JoinType join_type, const HashTableBuildDagMap &hashtable_build_dag_map, int device_count, int shard_count, const std::vector< std::vector< Fragmenter_Namespace::FragmentInfo >> &frags_for_device, Executor *executor)
const size_t table_tuple_count_
ThreadLocalIds thread_local_ids()
const std::vector< JoinColumn > join_columns
bool force_one_to_many_hash_join
static bool layoutRequiresAdditionalBuffers(HashType layout) noexcept
size_t g_bbox_intersect_max_table_size_bytes
size_t emitted_keys_count
bool tuneSmallerOneStep(const double step_overide)
HashTableProps(const size_t entry_count, const size_t emitted_keys_count, const size_t hash_table_size, const std::vector< double > &bucket_sizes)
static CompositeKeyInfo getCompositeKeyInfo(const std::vector< InnerOuter > &inner_outer_pairs, const Executor *executor, const std::vector< InnerOuterStringOpInfos > &inner_outer_string_op_infos_pairs={})
size_t getComponentBufferSize() const noexceptoverride