26 #include "../Shared/funcannotations.h"
27 #include "../Utils/ChunkIter.h"
30 #ifdef EXECUTE_INCLUDE
33 const uint64_t row_pos,
34 const uint32_t elem_log_sz) {
39 return ad.is_null ? 0 : ad.length >> elem_log_sz;
43 const uint64_t row_pos,
44 const uint32_t elem_log_sz,
45 const int32_t null_val) {
50 return ad.is_null ? null_val : ad.length >> elem_log_sz;
54 const uint64_t row_pos) {
63 const uint64_t row_pos) {
71 #define ARRAY_AT(type) \
72 extern "C" DEVICE RUNTIME_EXPORT type array_at_##type( \
73 int8_t* chunk_iter_, const uint64_t row_pos, const uint32_t elem_idx) { \
74 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
77 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
78 return reinterpret_cast<type*>(ad.pointer)[elem_idx]; \
90 #define VARLEN_ARRAY_AT(type) \
91 extern "C" DEVICE RUNTIME_EXPORT type varlen_array_at_##type( \
92 int8_t* chunk_iter_, const uint64_t row_pos, const uint32_t elem_idx) { \
93 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
96 ChunkIter_get_nth_varlen(chunk_iter, row_pos, &ad, &is_end); \
97 return reinterpret_cast<type*>(ad.pointer)[elem_idx]; \
100 VARLEN_ARRAY_AT(int8_t)
101 VARLEN_ARRAY_AT(int16_t)
102 VARLEN_ARRAY_AT(int32_t)
103 VARLEN_ARRAY_AT(int64_t)
104 VARLEN_ARRAY_AT(
float)
105 VARLEN_ARRAY_AT(
double)
107 #undef VARLEN_ARRAY_AT
109 #define VARLEN_NOTNULL_ARRAY_AT(type) \
110 extern "C" DEVICE RUNTIME_EXPORT type varlen_notnull_array_at_##type( \
111 int8_t* chunk_iter_, const uint64_t row_pos, const uint32_t elem_idx) { \
112 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
115 ChunkIter_get_nth_varlen_notnull(chunk_iter, row_pos, &ad, &is_end); \
116 return reinterpret_cast<type*>(ad.pointer)[elem_idx]; \
119 VARLEN_NOTNULL_ARRAY_AT(int8_t)
120 VARLEN_NOTNULL_ARRAY_AT(int16_t)
121 VARLEN_NOTNULL_ARRAY_AT(int32_t)
122 VARLEN_NOTNULL_ARRAY_AT(int64_t)
123 VARLEN_NOTNULL_ARRAY_AT(
float)
124 VARLEN_NOTNULL_ARRAY_AT(
double)
126 #undef VARLEN_NOTNULL_ARRAY_AT
128 #define ARRAY_ANY(type, needle_type, oper_name, oper) \
129 extern "C" DEVICE RUNTIME_EXPORT bool array_any_##oper_name##_##type##_##needle_type( \
130 int8_t* chunk_iter_, \
131 const uint64_t row_pos, \
132 const needle_type needle, \
133 const type null_val) { \
134 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
137 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
138 const size_t elem_count = ad.length / sizeof(type); \
139 for (size_t i = 0; i < elem_count; ++i) { \
140 const needle_type val = reinterpret_cast<type*>(ad.pointer)[i]; \
141 if (val != null_val && val oper needle) { \
148 #define ARRAY_ALL(type, needle_type, oper_name, oper) \
149 extern "C" DEVICE RUNTIME_EXPORT bool array_all_##oper_name##_##type##_##needle_type( \
150 int8_t* chunk_iter_, \
151 const uint64_t row_pos, \
152 const needle_type needle, \
153 const type null_val) { \
154 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
157 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
158 const size_t elem_count = ad.length / sizeof(type); \
159 for (size_t i = 0; i < elem_count; ++i) { \
160 const needle_type val = reinterpret_cast<type*>(ad.pointer)[i]; \
161 if (!(val != null_val && val oper needle)) { \
168 #define ARRAY_ALL_ANY_ALL_TYPES(oper_name, oper, needle_type) \
169 ARRAY_ANY(int8_t, needle_type, oper_name, oper) \
170 ARRAY_ALL(int8_t, needle_type, oper_name, oper) \
171 ARRAY_ANY(int16_t, needle_type, oper_name, oper) \
172 ARRAY_ALL(int16_t, needle_type, oper_name, oper) \
173 ARRAY_ANY(int32_t, needle_type, oper_name, oper) \
174 ARRAY_ALL(int32_t, needle_type, oper_name, oper) \
175 ARRAY_ANY(int64_t, needle_type, oper_name, oper) \
176 ARRAY_ALL(int64_t, needle_type, oper_name, oper) \
177 ARRAY_ANY(float, needle_type, oper_name, oper) \
178 ARRAY_ALL(float, needle_type, oper_name, oper) \
179 ARRAY_ANY(double, needle_type, oper_name, oper) \
180 ARRAY_ALL(double, needle_type, oper_name, oper)
182 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int8_t)
183 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int8_t)
184 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int8_t)
185 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int8_t)
186 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int8_t)
187 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int8_t)
189 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int16_t)
190 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int16_t)
191 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int16_t)
192 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int16_t)
193 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int16_t)
194 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int16_t)
196 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int32_t)
197 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int32_t)
198 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int32_t)
199 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int32_t)
200 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int32_t)
201 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int32_t)
203 ARRAY_ALL_ANY_ALL_TYPES(eq, ==, int64_t)
204 ARRAY_ALL_ANY_ALL_TYPES(ne, !=, int64_t)
205 ARRAY_ALL_ANY_ALL_TYPES(lt, <, int64_t)
206 ARRAY_ALL_ANY_ALL_TYPES(le, <=, int64_t)
207 ARRAY_ALL_ANY_ALL_TYPES(gt, >, int64_t)
208 ARRAY_ALL_ANY_ALL_TYPES(ge, >=, int64_t)
210 ARRAY_ALL_ANY_ALL_TYPES(eq, ==,
float)
211 ARRAY_ALL_ANY_ALL_TYPES(ne, !=,
float)
212 ARRAY_ALL_ANY_ALL_TYPES(lt, <,
float)
213 ARRAY_ALL_ANY_ALL_TYPES(le, <=,
float)
214 ARRAY_ALL_ANY_ALL_TYPES(gt, >,
float)
215 ARRAY_ALL_ANY_ALL_TYPES(ge, >=,
float)
217 ARRAY_ALL_ANY_ALL_TYPES(eq, ==,
double)
218 ARRAY_ALL_ANY_ALL_TYPES(ne, !=,
double)
219 ARRAY_ALL_ANY_ALL_TYPES(lt, <,
double)
220 ARRAY_ALL_ANY_ALL_TYPES(le, <=,
double)
221 ARRAY_ALL_ANY_ALL_TYPES(gt, >,
double)
222 ARRAY_ALL_ANY_ALL_TYPES(ge, >=,
double)
224 #undef ARRAY_ALL_ANY_ALL_TYPES
228 #define ARRAY_AT_CHECKED(type) \
229 extern "C" DEVICE RUNTIME_EXPORT type array_at_##type##_checked( \
230 int8_t* chunk_iter_, \
231 const uint64_t row_pos, \
232 const int64_t elem_idx, \
233 const type null_val) { \
234 if (elem_idx <= 0) { \
237 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
240 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
241 if (ad.is_null || static_cast<size_t>(elem_idx) > ad.length / sizeof(type)) { \
244 return reinterpret_cast<type*>(ad.pointer)[elem_idx - 1]; \
247 ARRAY_AT_CHECKED(int8_t)
248 ARRAY_AT_CHECKED(int16_t)
249 ARRAY_AT_CHECKED(int32_t)
250 ARRAY_AT_CHECKED(int64_t)
251 ARRAY_AT_CHECKED(
float)
252 ARRAY_AT_CHECKED(
double)
254 #undef ARRAY_AT_CHECKED
257 int64_t element_size) {
259 int8_t* varlen_buffer =
260 reinterpret_cast<int8_t*
>(
checked_malloc((element_count + 1) * element_size));
261 return varlen_buffer;
268 fast_fixlen_array_size(int8_t* chunk_iter_,
const uint32_t elem_log_sz) {
275 const uint64_t row_pos) {
277 auto n =
static_cast<int>(row_pos);
283 const uint64_t row_pos) {
310 const double dval{val};
311 return *
reinterpret_cast<const int64_t*
>(may_alias_ptr(&dval));
315 return *
reinterpret_cast<const int64_t*
>(may_alias_ptr(&val));
318 #define COUNT_DISTINCT_ARRAY(type) \
319 extern "C" RUNTIME_EXPORT void agg_count_distinct_array_##type( \
320 int64_t* agg, int8_t* chunk_iter_, const uint64_t row_pos, const type null_val) { \
321 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
324 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
325 const size_t elem_count{ad.length / sizeof(type)}; \
326 for (size_t i = 0; i < elem_count; ++i) { \
327 const auto val = reinterpret_cast<type*>(ad.pointer)[i]; \
328 if (val != null_val) { \
329 reinterpret_cast<CountDistinctSet*>(*agg)->insert(elem_bitcast_##type(val)); \
334 COUNT_DISTINCT_ARRAY(int8_t)
335 COUNT_DISTINCT_ARRAY(int16_t)
336 COUNT_DISTINCT_ARRAY(int32_t)
337 COUNT_DISTINCT_ARRAY(int64_t)
338 COUNT_DISTINCT_ARRAY(
float)
339 COUNT_DISTINCT_ARRAY(
double)
341 #undef COUNT_DISTINCT_ARRAY
346 const int64_t string_dict_handle);
348 #define ARRAY_STR_ANY(type, oper_name, oper) \
349 extern "C" RUNTIME_EXPORT bool array_any_##oper_name##_str_##type( \
350 int8_t* chunk_iter_, \
351 const uint64_t row_pos, \
352 const char* needle_ptr, \
353 const uint32_t needle_len, \
354 const int64_t string_dict_handle, \
355 const type null_val) { \
356 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
359 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
360 const size_t elem_count = ad.length / sizeof(type); \
361 std::string needle_str(needle_ptr, needle_len); \
362 for (size_t i = 0; i < elem_count; ++i) { \
363 const type val = reinterpret_cast<type*>(ad.pointer)[i]; \
364 if (val != null_val) { \
365 uint64_t str_and_len = string_decompress(val, string_dict_handle); \
366 const char* str = reinterpret_cast<const char*>(str_and_len & 0xffffffffffff); \
367 const uint16_t len = str_and_len >> 48; \
368 std::string val_str(str, len); \
369 if (val_str oper needle_str) { \
377 #define ARRAY_STR_ALL(type, oper_name, oper) \
378 extern "C" RUNTIME_EXPORT bool array_all_##oper_name##_str_##type( \
379 int8_t* chunk_iter_, \
380 const uint64_t row_pos, \
381 const char* needle_ptr, \
382 const uint32_t needle_len, \
383 const int64_t string_dict_handle, \
384 const type null_val) { \
385 ChunkIter* chunk_iter = reinterpret_cast<ChunkIter*>(chunk_iter_); \
388 ChunkIter_get_nth(chunk_iter, row_pos, &ad, &is_end); \
389 const size_t elem_count = ad.length / sizeof(type); \
390 std::string needle_str(needle_ptr, needle_len); \
391 for (size_t i = 0; i < elem_count; ++i) { \
392 const type val = reinterpret_cast<type*>(ad.pointer)[i]; \
393 if (val == null_val) { \
396 uint64_t str_and_len = string_decompress(val, string_dict_handle); \
397 const char* str = reinterpret_cast<const char*>(str_and_len & 0xffffffffffff); \
398 const uint16_t len = str_and_len >> 48; \
399 std::string val_str(str, len); \
400 if (!(val_str oper needle_str)) { \
407 #define ARRAY_STR_ALL_ANY_ALL_TYPES(oper_name, oper) \
408 ARRAY_STR_ANY(int8_t, oper_name, oper) \
409 ARRAY_STR_ALL(int8_t, oper_name, oper) \
410 ARRAY_STR_ANY(int16_t, oper_name, oper) \
411 ARRAY_STR_ALL(int16_t, oper_name, oper) \
412 ARRAY_STR_ANY(int32_t, oper_name, oper) \
413 ARRAY_STR_ALL(int32_t, oper_name, oper) \
414 ARRAY_STR_ANY(int64_t, oper_name, oper) \
415 ARRAY_STR_ALL(int64_t, oper_name, oper)
417 ARRAY_STR_ALL_ANY_ALL_TYPES(eq, ==)
418 ARRAY_STR_ALL_ANY_ALL_TYPES(ne, !=)
419 ARRAY_STR_ALL_ANY_ALL_TYPES(lt, <)
420 ARRAY_STR_ALL_ANY_ALL_TYPES(le, <=)
421 ARRAY_STR_ALL_ANY_ALL_TYPES(gt, >)
422 ARRAY_STR_ALL_ANY_ALL_TYPES(ge, >=)
424 #undef ARRAY_ALL_ANY_ALL_TYPES
430 #endif // EXECUTE_INCLUDE
DEVICE void ChunkIter_get_nth_point_coords(ChunkIter *it, int n, ArrayDatum *result, bool *is_end)
DEVICE void ChunkIter_get_nth(ChunkIter *it, int n, bool uncompress, VarlenDatum *result, bool *is_end)
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
EXTENSION_NOINLINE int8_t * allocate_varlen_buffer(int64_t element_count, int64_t element_size)
void * checked_malloc(const size_t size)
RUNTIME_EXPORT uint64_t string_decompress(const int32_t string_id, const int64_t string_dict_handle)