26 #include "../Shared/funcannotations.h"
27 #include "../Utils/ChunkIter.h"
30 #ifdef EXECUTE_INCLUDE
32 extern "C" DEVICE int32_t array_size(int8_t* chunk_iter_,
33 const uint64_t row_pos,
34 const uint32_t elem_log_sz) {
39 return ad.is_null ? 0 : ad.length >> elem_log_sz;
42 extern "C" DEVICE int32_t array_size_nullable(int8_t* chunk_iter_,
43 const uint64_t row_pos,
44 const uint32_t elem_log_sz,
45 const int32_t null_val) {
50 return ad.is_null ? null_val : ad.length >> elem_log_sz;
53 extern "C" DEVICE bool array_is_null(int8_t* chunk_iter_,
const uint64_t row_pos) {
61 extern "C" DEVICE bool point_coord_array_is_null(int8_t* chunk_iter_,
62 const uint64_t row_pos) {
70 #define ARRAY_AT(type) \
71 extern "C" DEVICE type array_at_##type( \
72 int8_t* chunk_iter_, const uint64_t row_pos, const uint32_t elem_idx) { \
73 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
76 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
77 return reinterpret_cast<type*>(ad.pointer)[elem_idx]; \
89 #define VARLEN_ARRAY_AT(type) \
90 extern "C" DEVICE type varlen_array_at_##type( \
91 int8_t* chunk_iter_, const uint64_t row_pos, const uint32_t elem_idx) { \
92 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
95 ChunkIter_get_nth_varlen(chunk_iter, row_pos, &ad, &is_end); \
96 return reinterpret_cast<type*>(ad.pointer)[elem_idx]; \
99 VARLEN_ARRAY_AT(int8_t)
100 VARLEN_ARRAY_AT(int16_t)
101 VARLEN_ARRAY_AT(int32_t)
102 VARLEN_ARRAY_AT(int64_t)
103 VARLEN_ARRAY_AT(
float)
104 VARLEN_ARRAY_AT(
double)
106 #undef VARLEN_ARRAY_AT
108 #define VARLEN_NOTNULL_ARRAY_AT(type) \
109 extern "C" DEVICE type varlen_notnull_array_at_##type( \
110 int8_t* chunk_iter_, const uint64_t row_pos, const uint32_t elem_idx) { \
111 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
114 ChunkIter_get_nth_varlen_notnull(chunk_iter, row_pos, &ad, &is_end); \
115 return reinterpret_cast<type*>(ad.pointer)[elem_idx]; \
118 VARLEN_NOTNULL_ARRAY_AT(int8_t)
119 VARLEN_NOTNULL_ARRAY_AT(int16_t)
120 VARLEN_NOTNULL_ARRAY_AT(int32_t)
121 VARLEN_NOTNULL_ARRAY_AT(int64_t)
122 VARLEN_NOTNULL_ARRAY_AT(
float)
123 VARLEN_NOTNULL_ARRAY_AT(
double)
125 #undef VARLEN_NOTNULL_ARRAY_AT
127 #define ARRAY_ANY(type, needle_type, oper_name, oper) \
128 extern "C" DEVICE bool array_any_##oper_name##_##type##_##needle_type( \
129 int8_t* chunk_iter_, \
130 const uint64_t row_pos, \
131 const needle_type needle, \
132 const type null_val) { \
133 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
136 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
137 const size_t elem_count = ad.length / sizeof(type); \
138 for (size_t i = 0; i < elem_count; ++i) { \
139 const needle_type val = reinterpret_cast<type*>(ad.pointer)[i]; \
140 if (val != null_val && val oper needle) { \
147 #define ARRAY_ALL(type, needle_type, oper_name, oper) \
148 extern "C" DEVICE bool array_all_##oper_name##_##type##_##needle_type( \
149 int8_t* chunk_iter_, \
150 const uint64_t row_pos, \
151 const needle_type needle, \
152 const type null_val) { \
153 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
156 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
157 const size_t elem_count = ad.length / sizeof(type); \
158 for (size_t i = 0; i < elem_count; ++i) { \
159 const needle_type val = reinterpret_cast<type*>(ad.pointer)[i]; \
160 if (!(val != null_val && val oper needle)) { \
167 #define ARRAY_ALL_ANY_ALL_TYPES(oper_name, oper, needle_type) \
168 ARRAY_ANY(int8_t, needle_type, oper_name, oper) \
169 ARRAY_ALL(int8_t, needle_type, oper_name, oper) \
170 ARRAY_ANY(int16_t, needle_type, oper_name, oper) \
171 ARRAY_ALL(int16_t, needle_type, oper_name, oper) \
172 ARRAY_ANY(int32_t, needle_type, oper_name, oper) \
173 ARRAY_ALL(int32_t, needle_type, oper_name, oper) \
174 ARRAY_ANY(int64_t, needle_type, oper_name, oper) \
175 ARRAY_ALL(int64_t, needle_type, oper_name, oper) \
176 ARRAY_ANY(float, needle_type, oper_name, oper) \
177 ARRAY_ALL(float, needle_type, oper_name, oper) \
178 ARRAY_ANY(double, needle_type, oper_name, oper) \
179 ARRAY_ALL(double, needle_type, oper_name, oper)
181 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int8_t)
182 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int8_t)
183 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int8_t)
184 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int8_t)
185 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int8_t)
186 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int8_t)
188 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int16_t)
189 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int16_t)
190 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int16_t)
191 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int16_t)
192 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int16_t)
193 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int16_t)
195 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int32_t)
196 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int32_t)
197 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int32_t)
198 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int32_t)
199 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int32_t)
200 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int32_t)
202 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int64_t)
203 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int64_t)
204 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int64_t)
205 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int64_t)
206 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int64_t)
207 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int64_t)
209 ARRAY_ALL_ANY_ALL_TYPES(eq, ==,
float)
210 ARRAY_ALL_ANY_ALL_TYPES(ne, !=,
float)
211 ARRAY_ALL_ANY_ALL_TYPES(lt, <,
float)
212 ARRAY_ALL_ANY_ALL_TYPES(le, <=,
float)
213 ARRAY_ALL_ANY_ALL_TYPES(gt, >,
float)
214 ARRAY_ALL_ANY_ALL_TYPES(ge, >=,
float)
216 ARRAY_ALL_ANY_ALL_TYPES(eq, ==,
double)
217 ARRAY_ALL_ANY_ALL_TYPES(ne, !=,
double)
218 ARRAY_ALL_ANY_ALL_TYPES(lt, <,
double)
219 ARRAY_ALL_ANY_ALL_TYPES(le, <=,
double)
220 ARRAY_ALL_ANY_ALL_TYPES(gt, >,
double)
221 ARRAY_ALL_ANY_ALL_TYPES(ge, >=,
double)
223 #undef ARRAY_ALL_ANY_ALL_TYPES
227 #define ARRAY_AT_CHECKED(type) \
228 extern "C" DEVICE type array_at_##type##_checked(int8_t* chunk_iter_, \
229 const uint64_t row_pos, \
230 const int64_t elem_idx, \
231 const type null_val) { \
232 if (elem_idx <= 0) { \
235 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
238 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
239 if (ad.is_null || static_cast<size_t>(elem_idx) > ad.length / sizeof(type)) { \
242 return reinterpret_cast<type*>(ad.pointer)[elem_idx - 1]; \
245 ARRAY_AT_CHECKED(int8_t)
246 ARRAY_AT_CHECKED(int16_t)
247 ARRAY_AT_CHECKED(int32_t)
248 ARRAY_AT_CHECKED(int64_t)
249 ARRAY_AT_CHECKED(
float)
250 ARRAY_AT_CHECKED(
double)
252 #undef ARRAY_AT_CHECKED
255 int64_t element_size) {
257 int8_t* varlen_buffer =
258 reinterpret_cast<int8_t*
>(
checked_malloc((element_count + 1) * element_size));
259 return varlen_buffer;
266 fast_fixlen_array_size(int8_t* chunk_iter_,
const uint32_t elem_log_sz) {
272 const uint64_t row_pos) {
274 auto n =
static_cast<int>(row_pos);
279 extern "C" DEVICE int8_t* array_buff(int8_t* chunk_iter_,
const uint64_t row_pos) {
291 extern "C" ALWAYS_INLINE int64_t elem_bitcast_int8_t(
const int8_t val) {
295 extern "C" ALWAYS_INLINE int64_t elem_bitcast_int16_t(
const int16_t val) {
299 extern "C" ALWAYS_INLINE int64_t elem_bitcast_int32_t(
const int32_t val) {
303 extern "C" ALWAYS_INLINE int64_t elem_bitcast_int64_t(
const int64_t val) {
307 extern "C" ALWAYS_INLINE int64_t elem_bitcast_float(
const float val) {
308 const double dval{val};
309 return *
reinterpret_cast<const int64_t*
>(may_alias_ptr(&dval));
312 extern "C" ALWAYS_INLINE int64_t elem_bitcast_double(
const double val) {
313 return *
reinterpret_cast<const int64_t*
>(may_alias_ptr(&val));
316 #define COUNT_DISTINCT_ARRAY(type) \
317 extern "C" void agg_count_distinct_array_##type( \
318 int64_t* agg, int8_t* chunk_iter_, const uint64_t row_pos, const type null_val) { \
319 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
322 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
323 const size_t elem_count{ad.length / sizeof(type)}; \
324 for (size_t i = 0; i < elem_count; ++i) { \
325 const auto val = reinterpret_cast<type*>(ad.pointer)[i]; \
326 if (val != null_val) { \
327 reinterpret_cast<std::set<int64_t>*>(*agg)->insert(elem_bitcast_##type(val)); \
332 COUNT_DISTINCT_ARRAY(int8_t)
333 COUNT_DISTINCT_ARRAY(int16_t)
334 COUNT_DISTINCT_ARRAY(int32_t)
335 COUNT_DISTINCT_ARRAY(int64_t)
336 COUNT_DISTINCT_ARRAY(
float)
337 COUNT_DISTINCT_ARRAY(
double)
339 #undef COUNT_DISTINCT_ARRAY
344 const int64_t string_dict_handle);
346 #define ARRAY_STR_ANY(type, oper_name, oper) \
347 extern "C" bool array_any_##oper_name##_str_##type(int8_t* chunk_iter_, \
348 const uint64_t row_pos, \
349 const char* needle_ptr, \
350 const uint32_t needle_len, \
351 const int64_t string_dict_handle, \
352 const type null_val) { \
353 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
356 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
357 const size_t elem_count = ad.length / sizeof(type); \
358 std::string needle_str(needle_ptr, needle_len); \
359 for (size_t i = 0; i < elem_count; ++i) { \
360 const type val = reinterpret_cast<type*>(ad.pointer)[i]; \
361 if (val != null_val) { \
362 uint64_t str_and_len = string_decompress(val, string_dict_handle); \
363 const char* str = reinterpret_cast<const char*>(str_and_len & 0xffffffffffff); \
364 const uint16_t len = str_and_len >> 48; \
365 std::string val_str(str, len); \
366 if (val_str oper needle_str) { \
374 #define ARRAY_STR_ALL(type, oper_name, oper) \
375 extern "C" bool array_all_##oper_name##_str_##type(int8_t* chunk_iter_, \
376 const uint64_t row_pos, \
377 const char* needle_ptr, \
378 const uint32_t needle_len, \
379 const int64_t string_dict_handle, \
380 const type null_val) { \
381 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
384 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
385 const size_t elem_count = ad.length / sizeof(type); \
386 std::string needle_str(needle_ptr, needle_len); \
387 for (size_t i = 0; i < elem_count; ++i) { \
388 const type val = reinterpret_cast<type*>(ad.pointer)[i]; \
389 if (val == null_val) { \
392 uint64_t str_and_len = string_decompress(val, string_dict_handle); \
393 const char* str = reinterpret_cast<const char*>(str_and_len & 0xffffffffffff); \
394 const uint16_t len = str_and_len >> 48; \
395 std::string val_str(str, len); \
396 if (!(val_str oper needle_str)) { \
403 #define ARRAY_STR_ALL_ANY_ALL_TYPES(oper_name, oper) \
404 ARRAY_STR_ANY(int8_t, oper_name, oper) \
405 ARRAY_STR_ALL(int8_t, oper_name, oper) \
406 ARRAY_STR_ANY(int16_t, oper_name, oper) \
407 ARRAY_STR_ALL(int16_t, oper_name, oper) \
408 ARRAY_STR_ANY(int32_t, oper_name, oper) \
409 ARRAY_STR_ALL(int32_t, oper_name, oper) \
410 ARRAY_STR_ANY(int64_t, oper_name, oper) \
411 ARRAY_STR_ALL(int64_t, oper_name, oper)
413 ARRAY_STR_ALL_ANY_ALL_TYPES(eq, ==)
414 ARRAY_STR_ALL_ANY_ALL_TYPES(ne, !=)
415 ARRAY_STR_ALL_ANY_ALL_TYPES(lt, <)
416 ARRAY_STR_ALL_ANY_ALL_TYPES(le, <=)
417 ARRAY_STR_ALL_ANY_ALL_TYPES(gt, >)
418 ARRAY_STR_ALL_ANY_ALL_TYPES(ge, >=)
420 #undef ARRAY_ALL_ANY_ALL_TYPES
426 #endif // EXECUTE_INCLUDE
EXTENSION_NOINLINE int8_t * allocate_varlen_buffer(int64_t element_count, int64_t element_size)
DEVICE void ChunkIter_get_nth_point_coords(ChunkIter *it, int n, ArrayDatum *result, bool *is_end)
DEVICE void ChunkIter_get_nth(ChunkIter *it, int n, bool uncompress, VarlenDatum *result, bool *is_end)
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
void * checked_malloc(const size_t size)
uint64_t string_decompress(const int32_t string_id, const int64_t string_dict_handle)