OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
RuntimeFunctions.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef QUERYENGINE_RUNTIMEFUNCTIONS_H
18 #define QUERYENGINE_RUNTIMEFUNCTIONS_H
19 
20 #include "Shared/funcannotations.h"
21 
22 #include <cassert>
23 #include <cstdint>
24 #include <ctime>
25 #include <limits>
26 #include <type_traits>
27 
28 extern "C" RUNTIME_EXPORT int64_t agg_sum(int64_t* agg, const int64_t val);
29 
30 extern "C" RUNTIME_EXPORT void agg_max(int64_t* agg, const int64_t val);
31 
32 extern "C" RUNTIME_EXPORT void agg_min(int64_t* agg, const int64_t val);
33 
34 extern "C" RUNTIME_EXPORT void agg_sum_double(int64_t* agg, const double val);
35 
36 extern "C" RUNTIME_EXPORT void agg_max_double(int64_t* agg, const double val);
37 
38 extern "C" RUNTIME_EXPORT void agg_min_double(int64_t* agg, const double val);
39 
40 extern "C" RUNTIME_EXPORT int32_t agg_sum_int32_skip_val(int32_t* agg,
41  const int32_t val,
42  const int32_t skip_val);
43 
44 extern "C" RUNTIME_EXPORT int64_t agg_sum_skip_val(int64_t* agg,
45  const int64_t val,
46  const int64_t skip_val);
47 
48 extern "C" RUNTIME_EXPORT void agg_max_skip_val(int64_t* agg,
49  const int64_t val,
50  const int64_t skip_val);
51 
52 extern "C" RUNTIME_EXPORT void agg_min_skip_val(int64_t* agg,
53  const int64_t val,
54  const int64_t skip_val);
55 
56 extern "C" RUNTIME_EXPORT void agg_sum_float_skip_val(int32_t* agg,
57  const float val,
58  const float skip_val);
59 
60 extern "C" RUNTIME_EXPORT void agg_sum_double_skip_val(int64_t* agg,
61  const double val,
62  const double skip_val);
63 
64 extern "C" RUNTIME_EXPORT void agg_max_double_skip_val(int64_t* agg,
65  const double val,
66  const double skip_val);
67 
68 extern "C" RUNTIME_EXPORT void agg_min_double_skip_val(int64_t* agg,
69  const double val,
70  const double skip_val);
71 
72 extern "C" RUNTIME_EXPORT int32_t agg_sum_int32(int32_t* agg, const int32_t val);
73 
74 extern "C" RUNTIME_EXPORT void agg_max_int32(int32_t* agg, const int32_t val);
75 extern "C" RUNTIME_EXPORT void agg_max_int16(int16_t* agg, const int16_t val);
76 extern "C" RUNTIME_EXPORT void agg_max_int8(int8_t* agg, const int8_t val);
77 
78 extern "C" RUNTIME_EXPORT void agg_min_int32(int32_t* agg, const int32_t val);
79 extern "C" RUNTIME_EXPORT void agg_min_int16(int16_t* agg, const int16_t val);
80 extern "C" RUNTIME_EXPORT void agg_min_int8(int8_t* agg, const int8_t val);
81 
82 extern "C" RUNTIME_EXPORT void agg_sum_float(int32_t* agg, const float val);
83 
84 extern "C" RUNTIME_EXPORT void agg_max_float(int32_t* agg, const float val);
85 
86 extern "C" RUNTIME_EXPORT void agg_min_float(int32_t* agg, const float val);
87 
88 extern "C" RUNTIME_EXPORT void agg_max_int32_skip_val(int32_t* agg,
89  const int32_t val,
90  const int32_t skip_val);
91 extern "C" RUNTIME_EXPORT void agg_max_int16_skip_val(int16_t* agg,
92  const int16_t val,
93  const int16_t skip_val);
94 extern "C" RUNTIME_EXPORT void agg_max_int8_skip_val(int8_t* agg,
95  const int8_t val,
96  const int8_t skip_val);
97 
98 extern "C" RUNTIME_EXPORT void agg_min_int32_skip_val(int32_t* agg,
99  const int32_t val,
100  const int32_t skip_val);
101 extern "C" RUNTIME_EXPORT void agg_min_int16_skip_val(int16_t* agg,
102  const int16_t val,
103  const int16_t skip_val);
104 extern "C" RUNTIME_EXPORT void agg_min_int8_skip_val(int8_t* agg,
105  const int8_t val,
106  const int8_t skip_val);
107 
108 extern "C" RUNTIME_EXPORT void agg_max_float_skip_val(int32_t* agg,
109  const float val,
110  const float skip_val);
111 
112 extern "C" RUNTIME_EXPORT void agg_min_float_skip_val(int32_t* agg,
113  const float val,
114  const float skip_val);
115 
116 extern "C" RUNTIME_EXPORT void agg_count_distinct_bitmap(int64_t* agg,
117  const int64_t val,
118  const int64_t min_val);
119 
120 #define EMPTY_KEY_64 std::numeric_limits<int64_t>::max()
121 #define EMPTY_KEY_32 std::numeric_limits<int32_t>::max()
122 #define EMPTY_KEY_16 std::numeric_limits<int16_t>::max()
123 #define EMPTY_KEY_8 std::numeric_limits<int8_t>::max()
124 
125 extern "C" RUNTIME_EXPORT uint32_t key_hash(const int64_t* key,
126  const uint32_t key_qw_count,
127  const uint32_t key_byte_width);
128 
129 extern "C" RUNTIME_EXPORT int64_t* get_group_value(
130  int64_t* groups_buffer,
131  const uint32_t groups_buffer_entry_count,
132  const int64_t* key,
133  const uint32_t key_count,
134  const uint32_t key_width,
135  const uint32_t row_size_quad);
136 
138 
139 extern "C" bool RUNTIME_EXPORT check_interrupt();
140 
141 extern "C" bool RUNTIME_EXPORT check_interrupt_init(unsigned command);
142 
144  int64_t* groups_buffer,
145  const uint32_t groups_buffer_entry_count,
146  const int64_t* key,
147  const uint32_t key_count,
148  const uint32_t key_width,
149  const uint32_t row_size_quad);
150 
151 extern "C" RUNTIME_EXPORT int64_t* get_group_value_columnar(
152  int64_t* groups_buffer,
153  const uint32_t groups_buffer_entry_count,
154  const int64_t* key,
155  const uint32_t key_qw_count);
156 
158  int64_t* groups_buffer,
159  const uint32_t groups_buffer_entry_count,
160  const int64_t* key,
161  const uint32_t key_qw_count);
162 
163 extern "C" RUNTIME_EXPORT int64_t* get_group_value_fast(int64_t* groups_buffer,
164  const int64_t key,
165  const int64_t min_key,
166  const int64_t bucket,
167  const uint32_t row_size_quad);
168 
170  int64_t* groups_buffer,
171  const int64_t key,
172  const int64_t orig_key,
173  const int64_t min_key,
174  const int64_t bucket,
175  const uint32_t row_size_quad);
176 
177 extern "C" RUNTIME_EXPORT uint32_t get_columnar_group_bin_offset(int64_t* key_base_ptr,
178  const int64_t key,
179  const int64_t min_key,
180  const int64_t bucket);
181 
183  int64_t* groups_buffer,
184  const uint32_t h,
185  const int64_t* key,
186  const uint32_t key_qw_count,
187  const uint32_t row_size_quad);
188 
190  int64_t* groups_buffer,
191  const uint32_t hashed_index,
192  const uint32_t row_size_quad);
193 
194 extern "C" RUNTIME_EXPORT int32_t* get_bucketized_hash_slot(
195  int32_t* buff,
196  const int64_t key,
197  const int64_t min_key,
198  const int64_t bucket_normalization = 1);
199 
200 extern "C" RUNTIME_EXPORT int32_t* get_hash_slot(int32_t* buff,
201  const int64_t key,
202  const int64_t min_key);
203 
204 extern "C" RUNTIME_EXPORT int32_t* get_hash_slot_sharded(
205  int32_t* buff,
206  const int64_t key,
207  const int64_t min_key,
208  const uint32_t entry_count_per_shard,
209  const uint32_t num_shards,
210  const uint32_t device_count);
211 
213  int32_t* buff,
214  const int64_t key,
215  const int64_t min_key,
216  const uint32_t entry_count_per_shard,
217  const uint32_t num_shards,
218  const uint32_t device_count,
219  const int64_t bucket_normalization);
220 
221 extern "C" RUNTIME_EXPORT int32_t* get_hash_slot_sharded_opt(
222  int32_t* buff,
223  const int64_t key,
224  const int64_t min_key,
225  const uint32_t entry_count_per_shard,
226  const uint32_t shard,
227  const uint32_t num_shards,
228  const uint32_t device_count);
229 
231  int32_t* buff,
232  const int64_t key,
233  const int64_t min_key,
234  const uint32_t entry_count_per_shard,
235  const uint32_t shard,
236  const uint32_t num_shards,
237  const uint32_t device_count,
238  const int64_t bucket_normalization);
239 
240 extern "C" RUNTIME_EXPORT int fill_one_to_one_hashtable(size_t idx,
241  int32_t* entry_ptr,
242  const int32_t invalid_slot_val);
243 
245  size_t idx,
246  int32_t* entry_ptr,
247  const int32_t invalid_slot_val);
248 
249 extern "C" RUNTIME_EXPORT void linear_probabilistic_count(uint8_t* bitmap,
250  const uint32_t bitmap_bytes,
251  const uint8_t* key_bytes,
252  const uint32_t key_len);
253 
254 // Regular fixed_width_*_decode are only available from the JIT,
255 // we need to call them for lazy fetch columns -- create wrappers.
256 
257 extern "C" RUNTIME_EXPORT int64_t
258 fixed_width_int_decode_noinline(const int8_t* byte_stream,
259  const int32_t byte_width,
260  const int64_t pos);
261 
262 extern "C" RUNTIME_EXPORT int64_t
263 fixed_width_unsigned_decode_noinline(const int8_t* byte_stream,
264  const int32_t byte_width,
265  const int64_t pos);
266 
268  const int8_t* byte_stream,
269  const int64_t pos);
270 
272  const int8_t* byte_stream,
273  const int64_t pos);
274 
275 extern "C" RUNTIME_EXPORT int64_t
276 fixed_width_small_date_decode_noinline(const int8_t* byte_stream,
277  const int32_t byte_width,
278  const int32_t null_val,
279  const int64_t ret_null_val,
280  const int64_t pos);
281 
282 extern "C" DEVICE NEVER_INLINE int64_t
283  SUFFIX(fixed_width_date_encode_noinline)(const int64_t cur_col_val,
284  const int32_t null_val,
285  const int64_t ret_null_val);
286 
287 extern "C" RUNTIME_EXPORT int8_t* extract_str_ptr_noinline(const uint64_t str_and_len);
288 
289 extern "C" RUNTIME_EXPORT int32_t extract_str_len_noinline(const uint64_t str_and_len);
290 
291 template <typename T = int64_t>
292 inline T get_empty_key() {
293  static_assert(std::is_same<T, int64_t>::value,
294  "Unsupported template parameter other than int64_t for now");
295  return EMPTY_KEY_64;
296 }
297 
298 template <>
299 inline int32_t get_empty_key() {
300  return EMPTY_KEY_32;
301 }
302 
303 #endif // QUERYENGINE_RUNTIMEFUNCTIONS_H
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int64_t * get_group_value_fast_with_original_key(int64_t *groups_buffer, const int64_t key, const int64_t orig_key, const int64_t min_key, const int64_t bucket, const uint32_t row_size_quad)
RUNTIME_EXPORT void agg_min_int8(int8_t *agg, const int8_t val)
RUNTIME_EXPORT void agg_max_int32(int32_t *agg, const int32_t val)
RUNTIME_EXPORT ALWAYS_INLINE int64_t * get_matching_group_value_perfect_hash(int64_t *groups_buffer, const uint32_t hashed_index, const int64_t *key, const uint32_t key_count, const uint32_t row_size_quad)
RUNTIME_EXPORT void agg_min_int16(int16_t *agg, const int16_t val)
RUNTIME_EXPORT void agg_max_int16(int16_t *agg, const int16_t val)
RUNTIME_EXPORT void agg_min_int32_skip_val(int32_t *agg, const int32_t val, const int32_t skip_val)
ALWAYS_INLINE DEVICE int32_t *SUFFIX() get_bucketized_hash_slot(int32_t *buff, const int64_t key, const int64_t min_key, const int64_t bucket_normalization)
Definition: JoinHashImpl.h:66
DEVICE NEVER_INLINE int64_t SUFFIX() fixed_width_int_decode_noinline(const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
Definition: DecodersImpl.h:91
RUNTIME_EXPORT ALWAYS_INLINE void agg_max_double(int64_t *agg, const double val)
RUNTIME_EXPORT ALWAYS_INLINE void agg_max(int64_t *agg, const int64_t val)
#define EMPTY_KEY_64
RUNTIME_EXPORT ALWAYS_INLINE void agg_min_float(int32_t *agg, const float val)
ALWAYS_INLINE DEVICE int32_t *SUFFIX() get_bucketized_hash_slot_sharded_opt(int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t shard, const uint32_t num_shards, const uint32_t device_count, const int64_t bucket_normalization)
Definition: JoinHashImpl.h:109
#define SUFFIX(name)
RUNTIME_EXPORT ALWAYS_INLINE void agg_count_distinct_bitmap(int64_t *agg, const int64_t val, const int64_t min_val)
RuntimeInterruptFlags
RUNTIME_EXPORT ALWAYS_INLINE DEVICE uint32_t get_columnar_group_bin_offset(int64_t *key_base_ptr, const int64_t key, const int64_t min_key, const int64_t bucket)
RUNTIME_EXPORT void agg_min_int8_skip_val(int8_t *agg, const int8_t val, const int8_t skip_val)
ALWAYS_INLINE DEVICE int SUFFIX() fill_hashtable_for_semi_join(size_t idx, int32_t *entry_ptr, const int32_t invalid_slot_val)
Definition: JoinHashImpl.h:54
RUNTIME_EXPORT void agg_max_int16_skip_val(int16_t *agg, const int16_t val, const int16_t skip_val)
RUNTIME_EXPORT ALWAYS_INLINE int64_t * get_matching_group_value_perfect_hash_keyless(int64_t *groups_buffer, const uint32_t hashed_index, const uint32_t row_size_quad)
RUNTIME_EXPORT void agg_sum_float_skip_val(int32_t *agg, const float val, const float skip_val)
#define DEVICE
__device__ bool check_interrupt()
RUNTIME_EXPORT void agg_sum_double_skip_val(int64_t *agg, const double val, const double skip_val)
ALWAYS_INLINE DEVICE int32_t *SUFFIX() get_hash_slot(int32_t *buff, const int64_t key, const int64_t min_key)
Definition: JoinHashImpl.h:74
RUNTIME_EXPORT NEVER_INLINE int32_t extract_str_len_noinline(const uint64_t str_and_len)
ALWAYS_INLINE DEVICE int32_t *SUFFIX() get_bucketized_hash_slot_sharded(int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t num_shards, const uint32_t device_count, const int64_t bucket_normalization)
Definition: JoinHashImpl.h:80
__device__ void linear_probabilistic_count(uint8_t *bitmap, const uint32_t bitmap_bytes, const uint8_t *key_bytes, const uint32_t key_len)
RUNTIME_EXPORT ALWAYS_INLINE void agg_min_double(int64_t *agg, const double val)
RUNTIME_EXPORT void agg_min_int16_skip_val(int16_t *agg, const int16_t val, const int16_t skip_val)
RUNTIME_EXPORT void agg_max_double_skip_val(int64_t *agg, const double val, const double skip_val)
DEVICE NEVER_INLINE int64_t SUFFIX() fixed_width_unsigned_decode_noinline(const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
Definition: DecodersImpl.h:98
RUNTIME_EXPORT ALWAYS_INLINE void agg_min(int64_t *agg, const int64_t val)
RUNTIME_EXPORT NEVER_INLINE DEVICE int64_t * get_group_value_columnar_with_watchdog(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_qw_count)
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_double(int64_t *agg, const double val)
RUNTIME_EXPORT NEVER_INLINE DEVICE int64_t * get_group_value(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad)
RUNTIME_EXPORT void agg_min_skip_val(int64_t *agg, const int64_t val, const int64_t skip_val)
#define RUNTIME_EXPORT
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32(int32_t *agg, const int32_t val)
RUNTIME_EXPORT NEVER_INLINE int8_t * extract_str_ptr_noinline(const uint64_t str_and_len)
DEVICE NEVER_INLINE int64_t SUFFIX() fixed_width_small_date_decode_noinline(const int8_t *byte_stream, const int32_t byte_width, const int32_t null_val, const int64_t ret_null_val, const int64_t pos)
Definition: DecodersImpl.h:149
ALWAYS_INLINE DEVICE int32_t *SUFFIX() get_hash_slot_sharded_opt(int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t shard, const uint32_t num_shards, const uint32_t device_count)
Definition: JoinHashImpl.h:124
RUNTIME_EXPORT void agg_max_int8(int8_t *agg, const int8_t val)
RUNTIME_EXPORT ALWAYS_INLINE void agg_max_float(int32_t *agg, const float val)
RUNTIME_EXPORT void agg_max_int8_skip_val(int8_t *agg, const int8_t val, const int8_t skip_val)
ALWAYS_INLINE DEVICE int32_t *SUFFIX() get_hash_slot_sharded(int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t num_shards, const uint32_t device_count)
Definition: JoinHashImpl.h:95
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int64_t * get_group_value_fast(int64_t *groups_buffer, const int64_t key, const int64_t min_key, const int64_t bucket, const uint32_t row_size_quad)
#define NEVER_INLINE
RUNTIME_EXPORT void agg_max_int32_skip_val(int32_t *agg, const int32_t val, const int32_t skip_val)
RUNTIME_EXPORT void agg_min_double_skip_val(int64_t *agg, const double val, const double skip_val)
RUNTIME_EXPORT ALWAYS_INLINE DEVICE uint32_t key_hash(const int64_t *key, const uint32_t key_count, const uint32_t key_byte_width)
ALWAYS_INLINE DEVICE int SUFFIX() fill_one_to_one_hashtable(size_t idx, int32_t *entry_ptr, const int32_t invalid_slot_val)
Definition: JoinHashImpl.h:44
RUNTIME_EXPORT void agg_max_float_skip_val(int32_t *agg, const float val, const float skip_val)
__device__ T get_empty_key()
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum_skip_val(int64_t *agg, const int64_t val, const int64_t skip_val)
DEVICE NEVER_INLINE float SUFFIX() fixed_width_float_decode_noinline(const int8_t *byte_stream, const int64_t pos)
Definition: DecodersImpl.h:121
DEVICE NEVER_INLINE double SUFFIX() fixed_width_double_decode_noinline(const int8_t *byte_stream, const int64_t pos)
Definition: DecodersImpl.h:134
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32_skip_val(int32_t *agg, const int32_t val, const int32_t skip_val)
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_float(int32_t *agg, const float val)
RUNTIME_EXPORT bool check_interrupt_init(unsigned command)
RUNTIME_EXPORT void agg_min_float_skip_val(int32_t *agg, const float val, const float skip_val)
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum(int64_t *agg, const int64_t val)
RUNTIME_EXPORT void agg_max_skip_val(int64_t *agg, const int64_t val, const int64_t skip_val)
RUNTIME_EXPORT NEVER_INLINE DEVICE int64_t * get_group_value_columnar(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_qw_count)
RUNTIME_EXPORT NEVER_INLINE DEVICE int64_t * get_group_value_with_watchdog(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad)
#define EMPTY_KEY_32
RUNTIME_EXPORT void agg_min_int32(int32_t *agg, const int32_t val)
DEVICE NEVER_INLINE int64_t SUFFIX() fixed_width_date_encode_noinline(const int64_t cur_col_val, const int32_t ret_null_val, const int64_t null_val)
Definition: DecodersImpl.h:166