OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Namespaces | Macros | Enumerations | Functions
RuntimeFunctions.cpp File Reference
#include "RuntimeFunctions.h"
#include "BufferCompaction.h"
#include "DecisionTreeEntry.h"
#include "HyperLogLogRank.h"
#include "MurmurHash.h"
#include "Shared/Datum.h"
#include "Shared/quantile.h"
#include "TypePunning.h"
#include "Utils/SegmentTreeUtils.h"
#include <atomic>
#include <chrono>
#include <cmath>
#include <cstring>
#include <functional>
#include <thread>
#include <tuple>
#include "DecodersImpl.h"
#include "GeoOpsRuntime.cpp"
#include "GroupByRuntime.cpp"
#include "JoinHashTable/Runtime/JoinHashTableQueryRuntime.cpp"
#include "TopKRuntime.cpp"
+ Include dependency graph for RuntimeFunctions.cpp:

Go to the source code of this file.

Namespaces

 anonymous_namespace{RuntimeFunctions.cpp}
 

Macros

#define DEF_ARITH_NULLABLE(type, null_type, opname, opsym)
 
#define DEF_ARITH_NULLABLE_LHS(type, null_type, opname, opsym)
 
#define DEF_ARITH_NULLABLE_RHS(type, null_type, opname, opsym)
 
#define DEF_CMP_NULLABLE(type, null_type, opname, opsym)
 
#define DEF_CMP_NULLABLE_LHS(type, null_type, opname, opsym)
 
#define DEF_CMP_NULLABLE_RHS(type, null_type, opname, opsym)
 
#define DEF_SAFE_DIV_NULLABLE(type, null_type, opname)
 
#define DEF_BINARY_NULLABLE_ALL_OPS(type, null_type)
 
#define DEF_MAP_STRING_TO_DATUM(value_type, value_name)
 
#define DEF_UMINUS_NULLABLE(type, null_type)
 
#define DEF_CAST_NULLABLE(from_type, to_type)
 
#define DEF_CAST_SCALED_NULLABLE(from_type, to_type)
 
#define DEF_CAST_NULLABLE_BIDIR(type1, type2)
 
#define DEF_ROUND_NULLABLE(from_type, to_type)
 
#define GPU_RT_STUB   NEVER_INLINE __attribute__((optnone))
 
#define DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(value_type, oper_name)
 
#define DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME_ALL_TYPES(oper_name)
 
#define DEF_RANGE_MODE_FRAME_LOWER_BOUND(target_val_type, col_type, null_type, opname, opsym)
 
#define DEF_RANGE_MODE_FRAME_UPPER_BOUND(target_val_type, col_type, null_type, opname, opsym)
 
#define DEF_GET_VALUE_IN_FRAME(col_type, logical_type)
 
#define DEF_SEARCH_AGGREGATION_TREE(agg_value_type)
 
#define DEF_SEARCH_DERIVED_AGGREGATION_TREE(agg_value_type)
 
#define DEF_HANDLE_NULL_FOR_WINDOW_FRAMING_AGG(agg_type, null_type)
 
#define DEF_FILL_MISSING_VALUE(col_type)
 
#define DEF_AGG_MAX_INT(n)
 
#define DEF_AGG_MIN_INT(n)
 
#define DEF_AGG_ID_INT(n)
 
#define DEF_CHECKED_SINGLE_AGG_ID_INT(n)
 
#define DEF_WRITE_PROJECTION_INT(n)
 
#define DEF_SKIP_AGG_ADD(base_agg_func)
 
#define DEF_SKIP_AGG(base_agg_func)
 
#define DATA_T   int64_t
 
#define DATA_T   int32_t
 
#define DATA_T   int16_t
 
#define DATA_T   int8_t
 
#define DEF_SKIP_AGG(base_agg_func)
 
#define DEF_SKIP_IF_AGG(skip_agg_func, base_agg_func)
 
#define DATA_T   double
 
#define ADDR_T   int64_t
 
#define DATA_T   float
 
#define ADDR_T   int32_t
 
#define DEF_SHARED_AGG_RET_STUBS(base_agg_func)
 
#define DEF_SHARED_AGG_STUBS(base_agg_func)
 

Enumerations

enum  anonymous_namespace{RuntimeFunctions.cpp}::AggFuncType { anonymous_namespace{RuntimeFunctions.cpp}::AggFuncType::MIN, anonymous_namespace{RuntimeFunctions.cpp}::AggFuncType::MAX, anonymous_namespace{RuntimeFunctions.cpp}::AggFuncType::SUM }
 

Functions

RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		scale_decimal_up (const int64_t operand, const uint64_t scale, const int64_t operand_null_val, const int64_t result_null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		scale_decimal_down_nullable (const int64_t operand, const int64_t scale, const int64_t null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		scale_decimal_down_not_nullable (const int64_t operand, const int64_t scale, const int64_t null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		floor_div_lhs (const int64_t dividend, const int64_t divisor)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		floor_div_nullable_lhs (const int64_t dividend, const int64_t divisor, const int64_t null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE int8_t logical_not (const int8_t operand, const int8_t null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE int8_t logical_and (const int8_t lhs, const int8_t rhs, const int8_t null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE int8_t logical_or (const int8_t lhs, const int8_t rhs, const int8_t null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint64_t 		agg_count (uint64_t *agg, const int64_t)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_count_distinct_bitmap (int64_t *agg, const int64_t val, const int64_t min_val, const int64_t bucket_size)
 
GPU_RT_STUB void agg_count_distinct_bitmap_gpu (int64_t *, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const uint64_t, const uint64_t)
 
RUNTIME_EXPORT NEVER_INLINE void agg_approximate_count_distinct (int64_t *agg, const int64_t key, const uint32_t b)
 
GPU_RT_STUB void agg_approximate_count_distinct_gpu (int64_t *, const int64_t, const uint32_t, const int64_t, const int64_t)
 
RUNTIME_EXPORT ALWAYS_INLINE int8_t bit_is_set (const int8_t *bitset, const int64_t val, const int64_t min_val, const int64_t max_val, const int64_t null_val, const int8_t null_bool_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		compute_int64_t_lower_bound (const int64_t entry_cnt, const int64_t target_value, const int64_t *col_buf)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		get_valid_buf_start_pos (const int64_t null_start_pos, const int64_t null_end_pos)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		get_valid_buf_end_pos (const int64_t num_elems, const int64_t null_start_pos, const int64_t null_end_pos)
 
template<typename T , typename Comparator >
int64_t compute_current_row_idx_in_frame (const int64_t num_elems, const int64_t cur_row_idx, const T *col_buf, const int32_t *partition_rowid_buf, const int64_t *ordered_index_buf, const T null_val, const bool nulls_first, const int64_t null_start_pos, const int64_t null_end_pos, Comparator cmp)
 
template<typename TARGET_VAL_TYPE , typename COL_TYPE , typename NULL_TYPE >
int64_t compute_lower_bound_from_ordered_partition_index (const int64_t num_elems, const TARGET_VAL_TYPE target_val, const COL_TYPE *col_buf, const int32_t *partition_rowid_buf, const int64_t *ordered_index_buf, const NULL_TYPE null_val, const bool nulls_first, const int64_t null_start_offset, const int64_t null_end_offset)
 
template<typename TARGET_VAL_TYPE , typename COL_TYPE , typename NULL_TYPE >
int64_t compute_upper_bound_from_ordered_partition_index (const int64_t num_elems, const TARGET_VAL_TYPE target_val, const COL_TYPE *col_buf, const int32_t *partition_rowid_buf, const int64_t *ordered_index_buf, const NULL_TYPE null_val, const bool nulls_first, const int64_t null_start_offset, const int64_t null_end_offset)
 
template<typename COL_TYPE , typename LOGICAL_TYPE >
LOGICAL_TYPE get_value_in_window_frame (const int64_t target_row_idx_in_frame, const int64_t frame_start_offset, const int64_t frame_end_offset, const COL_TYPE *col_buf, const int32_t *partition_rowid_buf, const int64_t *ordered_index_buf, const LOGICAL_TYPE logical_null_val, const LOGICAL_TYPE col_null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		encode_date (int64_t decoded_val, int64_t null_val, int64_t multiplier)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		compute_row_mode_start_index_sub (int64_t candidate_index, int64_t current_partition_start_offset, int64_t frame_bound)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		compute_row_mode_start_index_add (int64_t candidate_index, int64_t current_partition_start_offset, int64_t frame_bound, int64_t num_current_partition_elem)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		compute_row_mode_end_index_sub (int64_t candidate_index, int64_t current_partition_start_offset, int64_t frame_bound)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		compute_row_mode_end_index_add (int64_t candidate_index, int64_t current_partition_start_offset, int64_t frame_bound, int64_t num_current_partition_elem)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t * 		get_integer_aggregation_tree (int64_t **aggregation_trees, size_t partition_idx)
 
RUNTIME_EXPORT ALWAYS_INLINE
double * 		get_double_aggregation_tree (int64_t **aggregation_trees, size_t partition_idx)
 
RUNTIME_EXPORT ALWAYS_INLINE
SumAndCountPair< int64_t > * 		get_integer_derived_aggregation_tree (int64_t **aggregation_trees, size_t partition_idx)
 
RUNTIME_EXPORT ALWAYS_INLINE
SumAndCountPair< double > * 		get_double_derived_aggregation_tree (int64_t **aggregation_trees, size_t partition_idx)
 
RUNTIME_EXPORT ALWAYS_INLINE size_t getStartOffsetForSegmentTreeTraversal (size_t level, size_t tree_fanout)
 
template<AggFuncType AGG_FUNC_TYPE, typename AGG_TYPE >
AGG_TYPE anonymous_namespace{RuntimeFunctions.cpp}::agg_func (AGG_TYPE const lhs, AGG_TYPE const rhs)
 
template<AggFuncType AGG_FUNC_TYPE, typename AGG_TYPE >
AGG_TYPE compute_window_func_via_aggregation_tree (AGG_TYPE *aggregation_tree_for_partition, size_t query_range_start_idx, size_t query_range_end_idx, size_t leaf_level, size_t tree_fanout, AGG_TYPE init_val, AGG_TYPE invalid_val, AGG_TYPE null_val)
 
template<typename AGG_VALUE_TYPE >
void compute_derived_aggregates (SumAndCountPair< AGG_VALUE_TYPE > *aggregation_tree_for_partition, SumAndCountPair< AGG_VALUE_TYPE > &res, size_t query_range_start_idx, size_t query_range_end_idx, size_t leaf_level, size_t tree_fanout, AGG_VALUE_TYPE invalid_val, AGG_VALUE_TYPE null_val)
 
template<typename T >
T fill_missing_value (int64_t const cur_idx, T const null_val, T *const col_buf, int64_t const num_elems_in_partition, int32_t *const partition_rowid_buf, int64_t *const ordered_index_buf, bool const is_forward_fill)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		agg_sum (int64_t *agg, const int64_t val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		agg_sum_if (int64_t *agg, const int64_t val, const int8_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_max (int64_t *agg, const int64_t val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_min (int64_t *agg, const int64_t val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_id (int64_t *agg, const int64_t val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int8_t * 		agg_id_varlen (int8_t *varlen_buffer, const int64_t offset, const int8_t *value, const int64_t size_bytes)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		checked_single_agg_id (int64_t *agg, const int64_t val, const int64_t null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_count_distinct_bitmap_skip_val (int64_t *agg, const int64_t val, const int64_t min_val, const int64_t bucket_size, const int64_t skip_val)
 
GPU_RT_STUB void agg_count_distinct_bitmap_skip_val_gpu (int64_t *, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const int64_t, const uint64_t, const uint64_t)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint32_t 		agg_count_int32 (uint32_t *agg, const int32_t)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint32_t 		agg_count_if_int32 (uint32_t *agg, const int32_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		agg_sum_int32 (int32_t *agg, const int32_t val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		agg_sum_if_int32 (int32_t *agg, const int32_t val, const int8_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		agg_sum_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		agg_sum_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		agg_sum_if_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val, const int8_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		agg_sum_if_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val, const int8_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint64_t 		agg_count_if (uint64_t *agg, const int64_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint64_t 		agg_count_skip_val (uint64_t *agg, const int64_t val, const int64_t skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint64_t 		agg_count_if_skip_val (uint64_t *agg, const int64_t cond, const int64_t skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint32_t 		agg_count_int32_skip_val (uint32_t *agg, const int32_t val, const int32_t skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint32_t 		agg_count_if_int32_skip_val (uint32_t *agg, const int32_t cond, const int32_t skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint64_t 		agg_count_double (uint64_t *agg, const double val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_double (int64_t *agg, const double val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_if_double (int64_t *agg, const double val, const int8_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_max_double (int64_t *agg, const double val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_min_double (int64_t *agg, const double val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_id_double (int64_t *agg, const double val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		checked_single_agg_id_double (int64_t *agg, const double val, const double null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint32_t 		agg_count_float (uint32_t *agg, const float val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_float (int32_t *agg, const float val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_if_sum_float (int32_t *agg, const float val, const int8_t cond)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_max_float (int32_t *agg, const float val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_min_float (int32_t *agg, const float val)
 
RUNTIME_EXPORT ALWAYS_INLINE void agg_id_float (int32_t *agg, const float val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		checked_single_agg_id_float (int32_t *agg, const float val, const float null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint64_t 		agg_count_double_skip_val (uint64_t *agg, const double val, const double skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
uint32_t 		agg_count_float_skip_val (uint32_t *agg, const float val, const float skip_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		decimal_floor (const int64_t x, const int64_t scale)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		decimal_ceil (const int64_t x, const int64_t scale)
 
GPU_RT_STUB int8_t * agg_id_varlen_shared (int8_t *varlen_buffer, const int64_t offset, const int8_t *value, const int64_t size_bytes)
 
GPU_RT_STUB int32_t checked_single_agg_id_shared (int64_t *agg, const int64_t val, const int64_t null_val)
 
GPU_RT_STUB int32_t checked_single_agg_id_int32_shared (int32_t *agg, const int32_t val, const int32_t null_val)
 
GPU_RT_STUB int32_t checked_single_agg_id_int16_shared (int16_t *agg, const int16_t val, const int16_t null_val)
 
GPU_RT_STUB int32_t checked_single_agg_id_int8_shared (int8_t *agg, const int8_t val, const int8_t null_val)
 
GPU_RT_STUB int32_t checked_single_agg_id_double_shared (int64_t *agg, const double val, const double null_val)
 
GPU_RT_STUB int32_t checked_single_agg_id_float_shared (int32_t *agg, const float val, const float null_val)
 
GPU_RT_STUB void agg_max_int16_skip_val_shared (int16_t *agg, const int16_t val, const int16_t skip_val)
 
GPU_RT_STUB void agg_max_int8_skip_val_shared (int8_t *agg, const int8_t val, const int8_t skip_val)
 
GPU_RT_STUB void agg_min_int16_skip_val_shared (int16_t *agg, const int16_t val, const int16_t skip_val)
 
GPU_RT_STUB void agg_min_int8_skip_val_shared (int8_t *agg, const int8_t val, const int8_t skip_val)
 
GPU_RT_STUB void agg_id_double_shared_slow (int64_t *agg, const double *val)
 
GPU_RT_STUB int64_t agg_sum_shared (int64_t *agg, const int64_t val)
 
GPU_RT_STUB int64_t agg_sum_if_shared (int64_t *agg, const int64_t val, const int8_t cond)
 
GPU_RT_STUB int64_t agg_sum_skip_val_shared (int64_t *agg, const int64_t val, const int64_t skip_val)
 
GPU_RT_STUB int64_t agg_sum_if_skip_val_shared (int64_t *agg, const int64_t val, const int64_t skip_val, const int8_t cond)
 
GPU_RT_STUB int32_t agg_sum_int32_shared (int32_t *agg, const int32_t val)
 
GPU_RT_STUB int32_t agg_sum_int32_skip_val_shared (int32_t *agg, const int32_t val, const int32_t skip_val)
 
GPU_RT_STUB void agg_sum_double_shared (int64_t *agg, const double val)
 
GPU_RT_STUB void agg_sum_double_skip_val_shared (int64_t *agg, const double val, const double skip_val)
 
GPU_RT_STUB void agg_sum_float_shared (int32_t *agg, const float val)
 
GPU_RT_STUB void agg_sum_float_skip_val_shared (int32_t *agg, const float val, const float skip_val)
 
GPU_RT_STUB int32_t agg_sum_if_int32_shared (int32_t *agg, const int32_t val, const int8_t cond)
 
GPU_RT_STUB int32_t agg_sum_if_int32_skip_val_shared (int32_t *agg, const int32_t val, const int32_t skip_val, const int8_t cond)
 
GPU_RT_STUB void agg_sum_if_double_shared (int64_t *agg, const double val, const int8_t cond)
 
GPU_RT_STUB void agg_sum_if_double_skip_val_shared (int64_t *agg, const double val, const double skip_val, const int8_t cond)
 
GPU_RT_STUB void agg_sum_if_float_shared (int32_t *agg, const float val, const int8_t cond)
 
GPU_RT_STUB void agg_sum_if_float_skip_val_shared (int32_t *agg, const float val, const float skip_val, const int8_t cond)
 
GPU_RT_STUB void force_sync ()
 
GPU_RT_STUB void sync_warp ()
 
GPU_RT_STUB void sync_warp_protected (int64_t thread_pos, int64_t row_count)
 
GPU_RT_STUB void sync_threadblock ()
 
GPU_RT_STUB void write_back_non_grouped_agg (int64_t *input_buffer, int64_t *output_buffer, const int32_t num_agg_cols)
 
RUNTIME_EXPORT NEVER_INLINE int32_t pos_start_impl (int32_t const *row_index_resume)
 
RUNTIME_EXPORT NEVER_INLINE int32_t group_buff_idx_impl ()
 
RUNTIME_EXPORT NEVER_INLINE int32_t pos_step_impl ()
 
GPU_RT_STUB int8_t thread_warp_idx (const int8_t warp_sz)
 
GPU_RT_STUB int64_t get_thread_index ()
 
GPU_RT_STUB int64_t * declare_dynamic_shared_memory ()
 
GPU_RT_STUB int64_t get_block_index ()
 
RUNTIME_EXPORT ALWAYS_INLINE void record_error_code (const int32_t err_code, int32_t *error_codes)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		get_error_code (int32_t *error_codes)
 
RUNTIME_EXPORT NEVER_INLINE
const int64_t * 		init_shared_mem_nop (const int64_t *groups_buffer, const int32_t groups_buffer_size)
 
RUNTIME_EXPORT NEVER_INLINE void write_back_nop (int64_t *dest, int64_t *src, const int32_t sz)
 
RUNTIME_EXPORT int64_t * init_shared_mem (const int64_t *global_groups_buffer, const int32_t groups_buffer_size)
 
RUNTIME_EXPORT NEVER_INLINE void init_group_by_buffer_gpu (int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_qw_count, const uint32_t agg_col_count, const bool keyless, const int8_t warp_size)
 
RUNTIME_EXPORT NEVER_INLINE void init_columnar_group_by_buffer_gpu (int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_qw_count, const uint32_t agg_col_count, const bool keyless, const bool blocks_share_memory, const int32_t frag_idx)
 
RUNTIME_EXPORT NEVER_INLINE void init_group_by_buffer_impl (int64_t *groups_buffer, const int64_t *init_vals, const uint32_t groups_buffer_entry_count, const uint32_t key_qw_count, const uint32_t agg_col_count, const bool keyless, const int8_t warp_size)
 
template<typename T >
ALWAYS_INLINE int64_t * get_matching_group_value (int64_t *groups_buffer, const uint32_t h, const T *key, const uint32_t key_count, const uint32_t row_size_quad)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t * 		get_matching_group_value (int64_t *groups_buffer, const uint32_t h, const int64_t *key, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad)
 
template<typename T >
ALWAYS_INLINE int32_t get_matching_group_value_columnar_slot (int64_t *groups_buffer, const uint32_t entry_count, const uint32_t h, const T *key, const uint32_t key_count)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		get_matching_group_value_columnar_slot (int64_t *groups_buffer, const uint32_t entry_count, const uint32_t h, const int64_t *key, const uint32_t key_count, const uint32_t key_width)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t * 		get_matching_group_value_columnar (int64_t *groups_buffer, const uint32_t h, const int64_t *key, const uint32_t key_qw_count, const size_t entry_count)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t * 		get_matching_group_value_perfect_hash (int64_t *groups_buffer, const uint32_t hashed_index, const int64_t *key, const uint32_t key_count, const uint32_t row_size_quad)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t * 		get_matching_group_value_perfect_hash_keyless (int64_t *groups_buffer, const uint32_t hashed_index, const uint32_t row_size_quad)
 
RUNTIME_EXPORT ALWAYS_INLINE void set_matching_group_value_perfect_hash_columnar (int64_t *groups_buffer, const uint32_t hashed_index, const int64_t *key, const uint32_t key_count, const uint32_t entry_count)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t * 		get_group_value_fast_keyless (int64_t *groups_buffer, const int64_t key, const int64_t min_key, const int64_t, const uint32_t row_size_quad)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t * 		get_group_value_fast_keyless_semiprivate (int64_t *groups_buffer, const int64_t key, const int64_t min_key, const int64_t, const uint32_t row_size_quad, const uint8_t thread_warp_idx, const uint8_t warp_size)
 
RUNTIME_EXPORT ALWAYS_INLINE
StringView 		string_pack (const int8_t *ptr, const int32_t len)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		char_length (const char *str, const int32_t str_len)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		char_length_nullable (const char *str, const int32_t str_len, const int32_t int_null)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		key_for_string_encoded (const int32_t str_id)
 
ALWAYS_INLINE DEVICE int32_t map_string_dict_id (const int32_t string_id, const int64_t translation_map_handle, const int32_t min_source_id)
 
ALWAYS_INLINE DEVICE double tree_model_reg_predict (const double *regressor_inputs, const int64_t decision_tree_table_handle, const int64_t decision_tree_offsets_handle, const int32_t num_regressors, const int32_t num_trees, const bool compute_avg, const double null_value)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE bool 		sample_ratio (const double proportion, const int64_t row_offset)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		width_bucket (const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		width_bucket_reversed (const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		width_bucket_nullable (const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count, const double null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
int32_t 		width_bucket_reversed_nullable (const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count, const double null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		width_bucket_no_oob_check (const double target_value, const double lower_bound, const double scale_factor)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		width_bucket_reversed_no_oob_check (const double target_value, const double lower_bound, const double scale_factor)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		width_bucket_expr (const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		width_bucket_expr_nullable (const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count, const double null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE int32_t 		width_bucket_expr_no_oob_check (const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count)
 
RUNTIME_EXPORT ALWAYS_INLINE
int64_t 		row_number_window_func (const int64_t output_buff, const int64_t pos)
 
RUNTIME_EXPORT ALWAYS_INLINE double percent_window_func (const int64_t output_buff, const int64_t pos)
 
RUNTIME_EXPORT ALWAYS_INLINE double load_double (const int64_t *agg)
 
RUNTIME_EXPORT ALWAYS_INLINE float load_float (const int32_t *agg)
 
RUNTIME_EXPORT ALWAYS_INLINE double load_avg_int (const int64_t *sum, const int64_t *count, const double null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE double load_avg_decimal (const int64_t *sum, const int64_t *count, const double null_val, const uint32_t scale)
 
RUNTIME_EXPORT ALWAYS_INLINE double load_avg_double (const int64_t *agg, const int64_t *count, const double null_val)
 
RUNTIME_EXPORT ALWAYS_INLINE double load_avg_float (const int32_t *agg, const int32_t *count, const double null_val)
 
RUNTIME_EXPORT NEVER_INLINE void linear_probabilistic_count (uint8_t *bitmap, const uint32_t bitmap_bytes, const uint8_t *key_bytes, const uint32_t key_len)
 
RUNTIME_EXPORT NEVER_INLINE void query_stub_hoisted_literals (int32_t *error_code, int32_t *total_matched, int64_t **out, const uint32_t frag_idx, const uint32_t *row_index_resume, const int8_t **col_buffers, const int8_t *literals, const int64_t *num_rows, const uint64_t *frag_row_offsets, const int32_t *max_matched, const int64_t *init_agg_value, const int64_t *join_hash_tables, const int8_t *row_func_mgr)
 
RUNTIME_EXPORT void multifrag_query_hoisted_literals (int32_t *error_code, int32_t *total_matched, int64_t **out, const uint32_t *num_fragments_ptr, const uint32_t *num_tables_ptr, const uint32_t *row_index_resume, const int8_t ***col_buffers, const int8_t *literals, const int64_t *num_rows, const uint64_t *frag_row_offsets, const int32_t *max_matched, const int64_t *init_agg_value, const int64_t *join_hash_tables, const int8_t *row_func_mgr)
 
RUNTIME_EXPORT NEVER_INLINE void query_stub (int32_t *error_code, int32_t *total_matched, int64_t **out, const uint32_t frag_idx, const uint32_t *row_index_resume, const int8_t **col_buffers, const int64_t *num_rows, const uint64_t *frag_row_offsets, const int32_t *max_matched, const int64_t *init_agg_value, const int64_t *join_hash_tables, const int8_t *row_func_mgr)
 
RUNTIME_EXPORT void multifrag_query (int32_t *error_code, int32_t *total_matched, int64_t **out, const uint32_t *num_fragments_ptr, const uint32_t *num_tables_ptr, const uint32_t *row_index_resume, const int8_t ***col_buffers, const int64_t *num_rows, const uint64_t *frag_row_offsets, const int32_t *max_matched, const int64_t *init_agg_value, const int64_t *join_hash_tables, const int8_t *row_func_mgr)
 
RUNTIME_EXPORT ALWAYS_INLINE
DEVICE bool 		check_interrupt ()
 
RUNTIME_EXPORT bool check_interrupt_init (unsigned command)
 

Macro Definition Documentation

#define ADDR_T   int64_t

Definition at line 1571 of file RuntimeFunctions.cpp.

#define ADDR_T   int32_t

Definition at line 1571 of file RuntimeFunctions.cpp.

#define DATA_T   int64_t

Definition at line 1570 of file RuntimeFunctions.cpp.

#define DATA_T   int32_t

Definition at line 1570 of file RuntimeFunctions.cpp.

#define DATA_T   int16_t

Definition at line 1570 of file RuntimeFunctions.cpp.

#define DATA_T   int8_t

Definition at line 1570 of file RuntimeFunctions.cpp.

#define DATA_T   double

Definition at line 1570 of file RuntimeFunctions.cpp.

#define DATA_T   float

Definition at line 1570 of file RuntimeFunctions.cpp.

#define DEF_AGG_ID_INT (   n)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void agg_id_int##n(int##n##_t* agg, \
const int##n##_t val) { \
*agg = val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
anonymous_namespace{Utm.h}::n
constexpr double n
Definition: Utm.h:38
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1247 of file RuntimeFunctions.cpp.

#define DEF_AGG_MAX_INT (   n)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void agg_max_int##n(int##n##_t* agg, \
const int##n##_t val) { \
*agg = std::max(*agg, val); \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
anonymous_namespace{Utm.h}::n
constexpr double n
Definition: Utm.h:38
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1225 of file RuntimeFunctions.cpp.

#define DEF_AGG_MIN_INT (   n)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void agg_min_int##n(int##n##_t* agg, \
const int##n##_t val) { \
*agg = std::min(*agg, val); \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
anonymous_namespace{Utm.h}::n
constexpr double n
Definition: Utm.h:38
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1236 of file RuntimeFunctions.cpp.

#define DEF_ARITH_NULLABLE (   type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE type opname##_##type##_nullable( \
const type lhs, const type rhs, const null_type null_val) { \
if (lhs != null_val && rhs != null_val) { \
return lhs opsym rhs; \
} \
return null_val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 45 of file RuntimeFunctions.cpp.

#define DEF_ARITH_NULLABLE_LHS (   type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE type opname##_##type##_nullable_lhs( \
const type lhs, const type rhs, const null_type null_val) { \
if (lhs != null_val) { \
return lhs opsym rhs; \
} \
return null_val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 54 of file RuntimeFunctions.cpp.

#define DEF_ARITH_NULLABLE_RHS (   type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE type opname##_##type##_nullable_rhs( \
const type lhs, const type rhs, const null_type null_val) { \
if (rhs != null_val) { \
return lhs opsym rhs; \
} \
return null_val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 63 of file RuntimeFunctions.cpp.

#define DEF_BINARY_NULLABLE_ALL_OPS (   type,
  null_type 
)

Definition at line 117 of file RuntimeFunctions.cpp.

#define DEF_CAST_NULLABLE (   from_type,
  to_type 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE to_type \
cast_##from_type##_to_##to_type##_nullable(const from_type operand, \
const from_type from_null_val, \
const to_type to_null_val) { \
return operand == from_null_val ? to_null_val : operand; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 261 of file RuntimeFunctions.cpp.

#define DEF_CAST_NULLABLE_BIDIR (   type1,
  type2 
)
Value:
DEF_CAST_NULLABLE(type1, type2) \
DEF_CAST_NULLABLE(type2, type1)
DEF_CAST_NULLABLE
#define DEF_CAST_NULLABLE(from_type, to_type)
Definition: RuntimeFunctions.cpp:261

Definition at line 278 of file RuntimeFunctions.cpp.

#define DEF_CAST_SCALED_NULLABLE (   from_type,
  to_type 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE to_type \
cast_##from_type##_to_##to_type##_scaled_nullable(const from_type operand, \
const from_type from_null_val, \
const to_type to_null_val, \
const to_type divider) { \
return operand == from_null_val ? to_null_val : operand / divider; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 269 of file RuntimeFunctions.cpp.

#define DEF_CHECKED_SINGLE_AGG_ID_INT (   n)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE int32_t checked_single_agg_id_int##n( \
int##n##_t* agg, const int##n##_t val, const int##n##_t null_val) { \
if (val == null_val) { \
return 0; \
} \
if (*agg == val) { \
return 0; \
} else if (*agg == null_val) { \
*agg = val; \
return 0; \
} else { \
/* see Execute::ERR_SINGLE_VALUE_FOUND_MULTIPLE_VALUES*/ \
return 15; \
} \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
anonymous_namespace{Utm.h}::n
constexpr double n
Definition: Utm.h:38
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1253 of file RuntimeFunctions.cpp.

#define DEF_CMP_NULLABLE (   type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE int8_t opname##_##type##_nullable( \
const type lhs, \
const type rhs, \
const null_type null_val, \
const int8_t null_bool_val) { \
if (lhs != null_val && rhs != null_val) { \
return lhs opsym rhs; \
} \
return null_bool_val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 72 of file RuntimeFunctions.cpp.

#define DEF_CMP_NULLABLE_LHS (   type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE int8_t opname##_##type##_nullable_lhs( \
const type lhs, \
const type rhs, \
const null_type null_val, \
const int8_t null_bool_val) { \
if (lhs != null_val) { \
return lhs opsym rhs; \
} \
return null_bool_val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 84 of file RuntimeFunctions.cpp.

#define DEF_CMP_NULLABLE_RHS (   type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE int8_t opname##_##type##_nullable_rhs( \
const type lhs, \
const type rhs, \
const null_type null_val, \
const int8_t null_bool_val) { \
if (rhs != null_val) { \
return lhs opsym rhs; \
} \
return null_bool_val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 96 of file RuntimeFunctions.cpp.

#define DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME (   value_type,
  oper_name 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE int64_t \
compute_##value_type##_##oper_name##_current_row_idx_in_frame( \
const int64_t num_elems, \
const int64_t cur_row_idx, \
const value_type* col_buf, \
const int32_t* partition_rowid_buf, \
const int64_t* ordered_index_buf, \
const value_type null_val, \
const bool nulls_first, \
const int64_t null_start_pos, \
const int64_t null_end_pos) { \
return compute_current_row_idx_in_frame<value_type>(num_elems, \
cur_row_idx, \
col_buf, \
partition_rowid_buf, \
ordered_index_buf, \
null_val, \
nulls_first, \
null_start_pos, \
null_end_pos, \
std::oper_name<value_type>{}); \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 500 of file RuntimeFunctions.cpp.

#define DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME_ALL_TYPES (   oper_name)
Value:
DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(int8_t, oper_name) \
DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(int16_t, oper_name) \
DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(int32_t, oper_name) \
DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(int64_t, oper_name) \
DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(float, oper_name) \
DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(double, oper_name)
DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME
#define DEF_COMPUTE_CURRENT_ROW_IDX_IN_FRAME(value_type, oper_name)
Definition: RuntimeFunctions.cpp:500

Definition at line 523 of file RuntimeFunctions.cpp.

#define DEF_FILL_MISSING_VALUE (   col_type)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE col_type fill_##col_type##_missing_value( \
int64_t const cur_row_idx_in_frame, \
col_type const null_val, \
col_type* const col_buf, \
int64_t const num_elems_in_partition, \
int32_t* const partition_rowid_buf, \
int64_t* const ordered_index_buf, \
bool const is_forward_fill) { \
return fill_missing_value<col_type>(cur_row_idx_in_frame, \
null_val, \
col_buf, \
num_elems_in_partition, \
partition_rowid_buf, \
ordered_index_buf, \
is_forward_fill); \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1105 of file RuntimeFunctions.cpp.

#define DEF_GET_VALUE_IN_FRAME (   col_type,
  logical_type 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE logical_type \
get_##col_type##_value_##logical_type##_type_in_frame( \
const int64_t target_row_idx_in_frame, \
const int64_t frame_start_offset, \
const int64_t frame_end_offset, \
const col_type* col_buf, \
const int32_t* partition_rowid_buf, \
const int64_t* ordered_index_buf, \
const logical_type logical_null_val, \
const logical_type col_null_val) { \
return get_value_in_window_frame<col_type, logical_type>(target_row_idx_in_frame, \
frame_start_offset, \
frame_end_offset, \
col_buf, \
partition_rowid_buf, \
ordered_index_buf, \
logical_null_val, \
col_null_val); \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 721 of file RuntimeFunctions.cpp.

#define DEF_HANDLE_NULL_FOR_WINDOW_FRAMING_AGG (   agg_type,
  null_type 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE agg_type \
handle_null_val_##agg_type##_##null_type##_window_framing_agg( \
agg_type res, null_type agg_null_val, agg_type input_col_null_val) { \
if (res == agg_null_val) { \
return input_col_null_val; \
} \
return res; \
}
run_benchmark_import.res
tuple res
Definition: run_benchmark_import.py:381
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1062 of file RuntimeFunctions.cpp.

#define DEF_MAP_STRING_TO_DATUM (   value_type,
  value_name 
)
Value:
extern "C" ALWAYS_INLINE DEVICE value_type map_string_to_datum_##value_name( \
const int32_t string_id, \
const int64_t translation_map_handle, \
const int32_t min_source_id) { \
const Datum* translation_map = \
reinterpret_cast<const Datum*>(translation_map_handle); \
const Datum& out_datum = translation_map[string_id - min_source_id]; \
return out_datum.value_name##val; \
}
DEVICE
#define DEVICE
Definition: funcannotations.h:20
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
Datum
Definition: Datum.h:69

Definition at line 178 of file RuntimeFunctions.cpp.

#define DEF_RANGE_MODE_FRAME_LOWER_BOUND (   target_val_type,
  col_type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE int64_t \
range_mode_##target_val_type##_##col_type##_##null_type##_##opname##_frame_lower_bound( \
const int64_t num_elems, \
const target_val_type target_value, \
const col_type* col_buf, \
const int32_t* partition_rowid_buf, \
const int64_t* ordered_index_buf, \
const int64_t frame_bound_val, \
const null_type null_val, \
const bool nulls_first, \
const int64_t null_start_pos, \
const int64_t null_end_pos) { \
if (target_value == null_val) { \
return null_start_pos; \
} \
target_val_type new_val = target_value opsym frame_bound_val; \
return compute_lower_bound_from_ordered_partition_index<target_val_type, \
col_type, \
null_type>( \
num_elems, \
new_val, \
col_buf, \
partition_rowid_buf, \
ordered_index_buf, \
null_val, \
nulls_first, \
null_start_pos, \
null_end_pos); \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
compute_lower_bound_from_ordered_partition_index
int64_t compute_lower_bound_from_ordered_partition_index(const int64_t num_elems, const TARGET_VAL_TYPE target_val, const COL_TYPE *col_buf, const int32_t *partition_rowid_buf, const int64_t *ordered_index_buf, const NULL_TYPE null_val, const bool nulls_first, const int64_t null_start_offset, const int64_t null_end_offset)
Definition: RuntimeFunctions.cpp:538
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 565 of file RuntimeFunctions.cpp.

#define DEF_RANGE_MODE_FRAME_UPPER_BOUND (   target_val_type,
  col_type,
  null_type,
  opname,
  opsym 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE int64_t \
range_mode_##target_val_type##_##col_type##_##null_type##_##opname##_frame_upper_bound( \
const int64_t num_elems, \
const target_val_type target_value, \
const col_type* col_buf, \
const int32_t* partition_rowid_buf, \
const int64_t* ordered_index_buf, \
const int64_t frame_bound_val, \
const null_type null_val, \
const bool nulls_first, \
const int64_t null_start_pos, \
const int64_t null_end_pos) { \
if (target_value == null_val) { \
return null_end_pos; \
} \
target_val_type new_val = target_value opsym frame_bound_val; \
return compute_upper_bound_from_ordered_partition_index<target_val_type, \
col_type, \
null_type>( \
num_elems, \
new_val, \
col_buf, \
partition_rowid_buf, \
ordered_index_buf, \
null_val, \
nulls_first, \
null_start_pos, \
null_end_pos); \
}
compute_upper_bound_from_ordered_partition_index
int64_t compute_upper_bound_from_ordered_partition_index(const int64_t num_elems, const TARGET_VAL_TYPE target_val, const COL_TYPE *col_buf, const int32_t *partition_rowid_buf, const int64_t *ordered_index_buf, const NULL_TYPE null_val, const bool nulls_first, const int64_t null_start_offset, const int64_t null_end_offset)
Definition: RuntimeFunctions.cpp:619
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 646 of file RuntimeFunctions.cpp.

#define DEF_ROUND_NULLABLE (   from_type,
  to_type 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE to_type \
cast_##from_type##_to_##to_type##_nullable(const from_type operand, \
const from_type from_null_val, \
const to_type to_null_val) { \
return operand == from_null_val \
? to_null_val \
: static_cast<to_type>(operand + (operand < from_type(0) \
? from_type(-0.5) \
: from_type(0.5))); \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 282 of file RuntimeFunctions.cpp.

#define DEF_SAFE_DIV_NULLABLE (   type,
  null_type,
  opname 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE type safe_div_##type( \
const type lhs, const type rhs, const null_type null_val) { \
if (lhs != null_val && rhs != null_val && rhs != 0) { \
return lhs / rhs; \
} \
return null_val; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 108 of file RuntimeFunctions.cpp.

#define DEF_SEARCH_AGGREGATION_TREE (   agg_value_type)

Definition at line 900 of file RuntimeFunctions.cpp.

#define DEF_SEARCH_DERIVED_AGGREGATION_TREE (   agg_value_type)

Definition at line 1021 of file RuntimeFunctions.cpp.

#define DEF_SHARED_AGG_RET_STUBS (   base_agg_func)

Definition at line 1600 of file RuntimeFunctions.cpp.

#define DEF_SHARED_AGG_STUBS (   base_agg_func)
Value:
extern "C" GPU_RT_STUB void base_agg_func##_shared(int64_t* agg, const int64_t val) {} \
\
extern "C" GPU_RT_STUB void base_agg_func##_skip_val_shared( \
int64_t* agg, const int64_t val, const int64_t skip_val) {} \
extern "C" GPU_RT_STUB void base_agg_func##_int32_shared(int32_t* agg, \
const int32_t val) {} \
extern "C" GPU_RT_STUB void base_agg_func##_int16_shared(int16_t* agg, \
const int16_t val) {} \
extern "C" GPU_RT_STUB void base_agg_func##_int8_shared(int8_t* agg, \
const int8_t val) {} \
\
extern "C" GPU_RT_STUB void base_agg_func##_int32_skip_val_shared( \
int32_t* agg, const int32_t val, const int32_t skip_val) {} \
\
extern "C" GPU_RT_STUB void base_agg_func##_double_shared(int64_t* agg, \
const double val) {} \
\
extern "C" GPU_RT_STUB void base_agg_func##_double_skip_val_shared( \
int64_t* agg, const double val, const double skip_val) {} \
extern "C" GPU_RT_STUB void base_agg_func##_float_shared(int32_t* agg, \
const float val) {} \
\
extern "C" GPU_RT_STUB void base_agg_func##_float_skip_val_shared( \
int32_t* agg, const float val, const float skip_val) {}
GPU_RT_STUB
#define GPU_RT_STUB
Definition: RuntimeFunctions.cpp:380

Definition at line 1639 of file RuntimeFunctions.cpp.

#define DEF_SKIP_AGG (   base_agg_func)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void base_agg_func##_skip_val( \
DATA_T* agg, const DATA_T val, const DATA_T skip_val) { \
if (val != skip_val) { \
const DATA_T old_agg = *agg; \
if (old_agg != skip_val) { \
base_agg_func(agg, val); \
} else { \
*agg = val; \
} \
} \
}
DATA_T
#define DATA_T
Definition: cuda_mapd_rt.cu:1065
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1540 of file RuntimeFunctions.cpp.

#define DEF_SKIP_AGG (   base_agg_func)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void base_agg_func##_skip_val( \
ADDR_T* agg, const DATA_T val, const DATA_T skip_val) { \
if (val != skip_val) { \
const ADDR_T old_agg = *agg; \
if (old_agg != *reinterpret_cast<const ADDR_T*>(may_alias_ptr(&skip_val))) { \
base_agg_func(agg, val); \
} else { \
*agg = *reinterpret_cast<const ADDR_T*>(may_alias_ptr(&val)); \
} \
} \
}
DATA_T
#define DATA_T
Definition: cuda_mapd_rt.cu:1065
ADDR_T
#define ADDR_T
Definition: cuda_mapd_rt.cu:1066
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1540 of file RuntimeFunctions.cpp.

#define DEF_SKIP_AGG_ADD (   base_agg_func)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void base_agg_func##_skip_val( \
DATA_T* agg, const DATA_T val, const DATA_T skip_val) { \
if (val != skip_val) { \
base_agg_func(agg, val); \
} \
}
DATA_T
#define DATA_T
Definition: cuda_mapd_rt.cu:1065
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1373 of file RuntimeFunctions.cpp.

#define DEF_SKIP_IF_AGG (   skip_agg_func,
  base_agg_func 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void skip_agg_func##_skip_val( \
ADDR_T* agg, const DATA_T val, const DATA_T skip_val, const int8_t cond) { \
if (cond) { \
base_agg_func##_skip_val(agg, val, skip_val); \
} \
}
DATA_T
#define DATA_T
Definition: cuda_mapd_rt.cu:1065
ADDR_T
#define ADDR_T
Definition: cuda_mapd_rt.cu:1066
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1553 of file RuntimeFunctions.cpp.

#define DEF_UMINUS_NULLABLE (   type,
  null_type 
)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE type uminus_##type##_nullable( \
const type operand, const null_type null_val) { \
return operand == null_val ? null_val : -operand; \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89

Definition at line 246 of file RuntimeFunctions.cpp.

#define DEF_WRITE_PROJECTION_INT (   n)
Value:
extern "C" RUNTIME_EXPORT ALWAYS_INLINE void write_projection_int##n( \
int8_t* slot_ptr, const int##n##_t val, const int64_t init_val) { \
if (val != init_val) { \
*reinterpret_cast<int##n##_t*>(slot_ptr) = val; \
} \
}
RUNTIME_EXPORT
#define RUNTIME_EXPORT
Definition: funcannotations.h:77
anonymous_namespace{Utm.h}::n
constexpr double n
Definition: Utm.h:38
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition: funcannotations.h:57

Definition at line 1281 of file RuntimeFunctions.cpp.

#define GPU_RT_STUB   NEVER_INLINE __attribute__((optnone))

Definition at line 380 of file RuntimeFunctions.cpp.

Function Documentation

RUNTIME_EXPORT NEVER_INLINE void agg_approximate_count_distinct ( int64_t *  agg,
const int64_t  key,
const uint32_t  b 
)

Definition at line 393 of file RuntimeFunctions.cpp.

References get_rank(), and MurmurHash64A().

393  {
394  const uint64_t hash = MurmurHash64A(&key, sizeof(key), 0);
395  const uint32_t index = hash >> (64 - b);
396  const uint8_t rank = get_rank(hash << b, 64 - b);
397  uint8_t* M = reinterpret_cast<uint8_t*>(*agg);
398  M[index] = std::max(M[index], rank);
399 }
get_rank
FORCE_INLINE uint8_t get_rank(uint64_t x, uint32_t b)
Definition: HyperLogLogRank.h:30
MurmurHash64A
RUNTIME_EXPORT NEVER_INLINE DEVICE uint64_t MurmurHash64A(const void *key, int len, uint64_t seed)
Definition: MurmurHash.cpp:27

+ Here is the call graph for this function:

GPU_RT_STUB void agg_approximate_count_distinct_gpu ( int64_t *  ,
const int64_t  ,
const uint32_t  ,
const int64_t  ,
const int64_t   
)

Definition at line 401 of file RuntimeFunctions.cpp.

405  {}
RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count ( uint64_t *  agg,
const int64_t   
)

Definition at line 361 of file RuntimeFunctions.cpp.

Referenced by agg_count_skip_val(), and anonymous_namespace{GroupByAndAggregate.cpp}::get_agg_count().

361  {
362  return (*agg)++;
363 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE void agg_count_distinct_bitmap ( int64_t *  agg,
const int64_t  val,
const int64_t  min_val,
const int64_t  bucket_size 
)

Definition at line 365 of file RuntimeFunctions.cpp.

Referenced by agg_count_distinct_bitmap_skip_val(), WindowFunctionContext::fillPartitionEnd(), WindowFunctionContext::fillPartitionStart(), anonymous_namespace{WindowContext.cpp}::index_to_partition_end(), and InValuesBitmap::InValuesBitmap().

369  {
370  uint64_t bitmap_idx = val - min_val;
371  if (1 < bucket_size) {
372  bitmap_idx /= static_cast<uint64_t>(bucket_size);
373  }
374  reinterpret_cast<int8_t*>(*agg)[bitmap_idx >> 3] |= (1 << (bitmap_idx & 7));
375 }

+ Here is the caller graph for this function:

GPU_RT_STUB void agg_count_distinct_bitmap_gpu ( int64_t *  ,
const int64_t  ,
const int64_t  ,
const int64_t  ,
const int64_t  ,
const int64_t  ,
const uint64_t  ,
const uint64_t   
)

Definition at line 383 of file RuntimeFunctions.cpp.

390  {}
RUNTIME_EXPORT ALWAYS_INLINE void agg_count_distinct_bitmap_skip_val ( int64_t *  agg,
const int64_t  val,
const int64_t  min_val,
const int64_t  bucket_size,
const int64_t  skip_val 
)

Definition at line 1181 of file RuntimeFunctions.cpp.

References agg_count_distinct_bitmap().

1186  {
1187  if (val != skip_val) {
1188  agg_count_distinct_bitmap(agg, val, min_val, bucket_size);
1189  }
1190 }
agg_count_distinct_bitmap
RUNTIME_EXPORT ALWAYS_INLINE void agg_count_distinct_bitmap(int64_t *agg, const int64_t val, const int64_t min_val, const int64_t bucket_size)
Definition: RuntimeFunctions.cpp:365

+ Here is the call graph for this function:

GPU_RT_STUB void agg_count_distinct_bitmap_skip_val_gpu ( int64_t *  ,
const int64_t  ,
const int64_t  ,
const int64_t  ,
const int64_t  ,
const int64_t  ,
const int64_t  ,
const uint64_t  ,
const uint64_t   
)

Definition at line 1192 of file RuntimeFunctions.cpp.

1200  {}
RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count_double ( uint64_t *  agg,
const double  val 
)

Definition at line 1419 of file RuntimeFunctions.cpp.

Referenced by agg_count_double_skip_val().

1420  {
1421  return (*agg)++;
1422 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count_double_skip_val ( uint64_t *  agg,
const double  val,
const double  skip_val 
)

Definition at line 1525 of file RuntimeFunctions.cpp.

References agg_count_double().

1525  {
1526  if (val != skip_val) {
1527  return agg_count_double(agg, val);
1528  }
1529  return *agg;
1530 }
agg_count_double
RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count_double(uint64_t *agg, const double val)
Definition: RuntimeFunctions.cpp:1419

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_float ( uint32_t *  agg,
const float  val 
)

Definition at line 1472 of file RuntimeFunctions.cpp.

Referenced by agg_count_float_skip_val().

1473  {
1474  return (*agg)++;
1475 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_float_skip_val ( uint32_t *  agg,
const float  val,
const float  skip_val 
)

Definition at line 1533 of file RuntimeFunctions.cpp.

References agg_count_float().

1533  {
1534  if (val != skip_val) {
1535  return agg_count_float(agg, val);
1536  }
1537  return *agg;
1538 }
agg_count_float
RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_float(uint32_t *agg, const float val)
Definition: RuntimeFunctions.cpp:1472

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count_if ( uint64_t *  agg,
const int64_t  cond 
)

Definition at line 1336 of file RuntimeFunctions.cpp.

Referenced by agg_count_if_skip_val().

1337  {
1338  return cond ? (*agg)++ : *agg;
1339 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_if_int32 ( uint32_t *  agg,
const int32_t  cond 
)

Definition at line 1207 of file RuntimeFunctions.cpp.

Referenced by agg_count_if_int32_skip_val().

1208  {
1209  return cond ? (*agg)++ : *agg;
1210 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_if_int32_skip_val ( uint32_t *  agg,
const int32_t  cond,
const int32_t  skip_val 
)

Definition at line 1366 of file RuntimeFunctions.cpp.

References agg_count_if_int32().

1366  {
1367  if (cond != skip_val) {
1368  return agg_count_if_int32(agg, cond);
1369  }
1370  return *agg;
1371 }
agg_count_if_int32
RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_if_int32(uint32_t *agg, const int32_t cond)
Definition: RuntimeFunctions.cpp:1207

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count_if_skip_val ( uint64_t *  agg,
const int64_t  cond,
const int64_t  skip_val 
)

Definition at line 1350 of file RuntimeFunctions.cpp.

References agg_count_if().

1350  {
1351  if (cond != skip_val) {
1352  return agg_count_if(agg, cond);
1353  }
1354  return *agg;
1355 }
agg_count_if
RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count_if(uint64_t *agg, const int64_t cond)
Definition: RuntimeFunctions.cpp:1336

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_int32 ( uint32_t *  agg,
const int32_t   
)

Definition at line 1202 of file RuntimeFunctions.cpp.

Referenced by agg_count_int32_skip_val().

1203  {
1204  return (*agg)++;
1205 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_int32_skip_val ( uint32_t *  agg,
const int32_t  val,
const int32_t  skip_val 
)

Definition at line 1358 of file RuntimeFunctions.cpp.

References agg_count_int32().

1358  {
1359  if (val != skip_val) {
1360  return agg_count_int32(agg, val);
1361  }
1362  return *agg;
1363 }
agg_count_int32
RUNTIME_EXPORT ALWAYS_INLINE uint32_t agg_count_int32(uint32_t *agg, const int32_t)
Definition: RuntimeFunctions.cpp:1202

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count_skip_val ( uint64_t *  agg,
const int64_t  val,
const int64_t  skip_val 
)

Definition at line 1342 of file RuntimeFunctions.cpp.

References agg_count().

1342  {
1343  if (val != skip_val) {
1344  return agg_count(agg, val);
1345  }
1346  return *agg;
1347 }
agg_count
RUNTIME_EXPORT ALWAYS_INLINE uint64_t agg_count(uint64_t *agg, const int64_t)
Definition: RuntimeFunctions.cpp:361

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE void agg_id ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1150 of file RuntimeFunctions.cpp.

1150  {
1151  *agg = val;
1152 }
RUNTIME_EXPORT ALWAYS_INLINE void agg_id_double ( int64_t *  agg,
const double  val 
)

Definition at line 1450 of file RuntimeFunctions.cpp.

1451  {
1452  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&val)));
1453 }
GPU_RT_STUB void agg_id_double_shared_slow ( int64_t *  agg,
const double *  val 
)

Definition at line 1731 of file RuntimeFunctions.cpp.

1731 {}
RUNTIME_EXPORT ALWAYS_INLINE void agg_id_float ( int32_t *  agg,
const float  val 
)

Definition at line 1503 of file RuntimeFunctions.cpp.

1503  {
1504  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&val)));
1505 }
RUNTIME_EXPORT ALWAYS_INLINE int8_t* agg_id_varlen ( int8_t *  varlen_buffer,
const int64_t  offset,
const int8_t *  value,
const int64_t  size_bytes 
)

Definition at line 1154 of file RuntimeFunctions.cpp.

1157  {
1158  for (auto i = 0; i < size_bytes; i++) {
1159  varlen_buffer[offset + i] = value[i];
1160  }
1161  return &varlen_buffer[offset];
1162 }
GPU_RT_STUB int8_t* agg_id_varlen_shared ( int8_t *  varlen_buffer,
const int64_t  offset,
const int8_t *  value,
const int64_t  size_bytes 
)

Definition at line 1671 of file RuntimeFunctions.cpp.

1674  {
1675  return nullptr;
1676 }
RUNTIME_EXPORT ALWAYS_INLINE void agg_if_sum_float ( int32_t *  agg,
const float  val,
const int8_t  cond 
)

Definition at line 1483 of file RuntimeFunctions.cpp.

References agg_sum_float().

1485  {
1486  if (cond) {
1487  agg_sum_float(agg, val);
1488  }
1489 }
agg_sum_float
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_float(int32_t *agg, const float val)
Definition: RuntimeFunctions.cpp:1477

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE void agg_max ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1142 of file RuntimeFunctions.cpp.

1142  {
1143  *agg = std::max(*agg, val);
1144 }
RUNTIME_EXPORT ALWAYS_INLINE void agg_max_double ( int64_t *  agg,
const double  val 
)

Definition at line 1438 of file RuntimeFunctions.cpp.

1439  {
1440  const auto r = std::max(*reinterpret_cast<const double*>(agg), val);
1441  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&r)));
1442 }
RUNTIME_EXPORT ALWAYS_INLINE void agg_max_float ( int32_t *  agg,
const float  val 
)

Definition at line 1491 of file RuntimeFunctions.cpp.

1492  {
1493  const auto r = std::max(*reinterpret_cast<const float*>(agg), val);
1494  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&r)));
1495 }
GPU_RT_STUB void agg_max_int16_skip_val_shared ( int16_t *  agg,
const int16_t  val,
const int16_t  skip_val 
)

Definition at line 1715 of file RuntimeFunctions.cpp.

1717  {}
GPU_RT_STUB void agg_max_int8_skip_val_shared ( int8_t *  agg,
const int8_t  val,
const int8_t  skip_val 
)

Definition at line 1719 of file RuntimeFunctions.cpp.

1721  {}
RUNTIME_EXPORT ALWAYS_INLINE void agg_min ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1146 of file RuntimeFunctions.cpp.

1146  {
1147  *agg = std::min(*agg, val);
1148 }
RUNTIME_EXPORT ALWAYS_INLINE void agg_min_double ( int64_t *  agg,
const double  val 
)

Definition at line 1444 of file RuntimeFunctions.cpp.

1445  {
1446  const auto r = std::min(*reinterpret_cast<const double*>(agg), val);
1447  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&r)));
1448 }
RUNTIME_EXPORT ALWAYS_INLINE void agg_min_float ( int32_t *  agg,
const float  val 
)

Definition at line 1497 of file RuntimeFunctions.cpp.

1498  {
1499  const auto r = std::min(*reinterpret_cast<const float*>(agg), val);
1500  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&r)));
1501 }
GPU_RT_STUB void agg_min_int16_skip_val_shared ( int16_t *  agg,
const int16_t  val,
const int16_t  skip_val 
)

Definition at line 1723 of file RuntimeFunctions.cpp.

1725  {}
GPU_RT_STUB void agg_min_int8_skip_val_shared ( int8_t *  agg,
const int8_t  val,
const int8_t  skip_val 
)

Definition at line 1727 of file RuntimeFunctions.cpp.

1729  {}
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1130 of file RuntimeFunctions.cpp.

Referenced by agg_sum_if(), and agg_sum_skip_val().

1130  {
1131  const auto old = *agg;
1132  *agg += val;
1133  return old;
1134 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_double ( int64_t *  agg,
const double  val 
)

Definition at line 1424 of file RuntimeFunctions.cpp.

Referenced by agg_sum_if_double().

1425  {
1426  const auto r = *reinterpret_cast<const double*>(agg) + val;
1427  *agg = *reinterpret_cast<const int64_t*>(may_alias_ptr(&r));
1428 }

+ Here is the caller graph for this function:

GPU_RT_STUB void agg_sum_double_shared ( int64_t *  agg,
const double  val 
)

Definition at line 1765 of file RuntimeFunctions.cpp.

1765 {}
GPU_RT_STUB void agg_sum_double_skip_val_shared ( int64_t *  agg,
const double  val,
const double  skip_val 
)

Definition at line 1767 of file RuntimeFunctions.cpp.

1769  {}
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_float ( int32_t *  agg,
const float  val 
)

Definition at line 1477 of file RuntimeFunctions.cpp.

Referenced by agg_if_sum_float().

1478  {
1479  const auto r = *reinterpret_cast<const float*>(agg) + val;
1480  *agg = *reinterpret_cast<const int32_t*>(may_alias_ptr(&r));
1481 }

+ Here is the caller graph for this function:

GPU_RT_STUB void agg_sum_float_shared ( int32_t *  agg,
const float  val 
)

Definition at line 1770 of file RuntimeFunctions.cpp.

1770 {}
GPU_RT_STUB void agg_sum_float_skip_val_shared ( int32_t *  agg,
const float  val,
const float  skip_val 
)

Definition at line 1772 of file RuntimeFunctions.cpp.

1774  {}
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum_if ( int64_t *  agg,
const int64_t  val,
const int8_t  cond 
)

Definition at line 1136 of file RuntimeFunctions.cpp.

References agg_sum().

1138  {
1139  return cond ? agg_sum(agg, val) : *agg;
1140 }
agg_sum
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum(int64_t *agg, const int64_t val)
Definition: RuntimeFunctions.cpp:1130

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_if_double ( int64_t *  agg,
const double  val,
const int8_t  cond 
)

Definition at line 1430 of file RuntimeFunctions.cpp.

References agg_sum_double().

1432  {
1433  if (cond) {
1434  agg_sum_double(agg, val);
1435  }
1436 }
agg_sum_double
RUNTIME_EXPORT ALWAYS_INLINE void agg_sum_double(int64_t *agg, const double val)
Definition: RuntimeFunctions.cpp:1424

+ Here is the call graph for this function:

GPU_RT_STUB void agg_sum_if_double_shared ( int64_t *  agg,
const double  val,
const int8_t  cond 
)

Definition at line 1789 of file RuntimeFunctions.cpp.

1791  {}
GPU_RT_STUB void agg_sum_if_double_skip_val_shared ( int64_t *  agg,
const double  val,
const double  skip_val,
const int8_t  cond 
)

Definition at line 1793 of file RuntimeFunctions.cpp.

1796  {}
GPU_RT_STUB void agg_sum_if_float_shared ( int32_t *  agg,
const float  val,
const int8_t  cond 
)

Definition at line 1797 of file RuntimeFunctions.cpp.

1799  {}
GPU_RT_STUB void agg_sum_if_float_skip_val_shared ( int32_t *  agg,
const float  val,
const float  skip_val,
const int8_t  cond 
)

Definition at line 1801 of file RuntimeFunctions.cpp.

1804  {}
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_if_int32 ( int32_t *  agg,
const int32_t  val,
const int8_t  cond 
)

Definition at line 1219 of file RuntimeFunctions.cpp.

References agg_sum_int32().

1221  {
1222  return cond ? agg_sum_int32(agg, val) : *agg;
1223 }
agg_sum_int32
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32(int32_t *agg, const int32_t val)
Definition: RuntimeFunctions.cpp:1212

+ Here is the call graph for this function:

GPU_RT_STUB int32_t agg_sum_if_int32_shared ( int32_t *  agg,
const int32_t  val,
const int8_t  cond 
)

Definition at line 1776 of file RuntimeFunctions.cpp.

1778  {
1779  return 0;
1780 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_if_int32_skip_val ( int32_t *  agg,
const int32_t  val,
const int32_t  skip_val,
const int8_t  cond 
)

Definition at line 1329 of file RuntimeFunctions.cpp.

References agg_sum_int32_skip_val().

1332  {
1333  return cond ? agg_sum_int32_skip_val(agg, val, skip_val) : *agg;
1334 }
agg_sum_int32_skip_val
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32_skip_val(int32_t *agg, const int32_t val, const int32_t skip_val)
Definition: RuntimeFunctions.cpp:1308

+ Here is the call graph for this function:

GPU_RT_STUB int32_t agg_sum_if_int32_skip_val_shared ( int32_t *  agg,
const int32_t  val,
const int32_t  skip_val,
const int8_t  cond 
)

Definition at line 1782 of file RuntimeFunctions.cpp.

1785  {
1786  return 0;
1787 }
GPU_RT_STUB int64_t agg_sum_if_shared ( int64_t *  agg,
const int64_t  val,
const int8_t  cond 
)

Definition at line 1737 of file RuntimeFunctions.cpp.

1739  {
1740  return 0;
1741 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum_if_skip_val ( int64_t *  agg,
const int64_t  val,
const int64_t  skip_val,
const int8_t  cond 
)

Definition at line 1321 of file RuntimeFunctions.cpp.

References agg_sum_skip_val().

1324  {
1325  return cond ? agg_sum_skip_val(agg, val, skip_val) : *agg;
1326 }
agg_sum_skip_val
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum_skip_val(int64_t *agg, const int64_t val, const int64_t skip_val)
Definition: RuntimeFunctions.cpp:1293

+ Here is the call graph for this function:

GPU_RT_STUB int64_t agg_sum_if_skip_val_shared ( int64_t *  agg,
const int64_t  val,
const int64_t  skip_val,
const int8_t  cond 
)

Definition at line 1749 of file RuntimeFunctions.cpp.

1752  {
1753  return 0;
1754 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32 ( int32_t *  agg,
const int32_t  val 
)

Definition at line 1212 of file RuntimeFunctions.cpp.

Referenced by agg_sum_if_int32(), and agg_sum_int32_skip_val().

1213  {
1214  const auto old = *agg;
1215  *agg += val;
1216  return old;
1217 }

+ Here is the caller graph for this function:

GPU_RT_STUB int32_t agg_sum_int32_shared ( int32_t *  agg,
const int32_t  val 
)

Definition at line 1755 of file RuntimeFunctions.cpp.

1755  {
1756  return 0;
1757 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32_skip_val ( int32_t *  agg,
const int32_t  val,
const int32_t  skip_val 
)

Definition at line 1308 of file RuntimeFunctions.cpp.

References agg_sum_int32().

Referenced by agg_sum_if_int32_skip_val().

1308  {
1309  const auto old = *agg;
1310  if (val != skip_val) {
1311  if (old != skip_val) {
1312  return agg_sum_int32(agg, val);
1313  } else {
1314  *agg = val;
1315  }
1316  }
1317  return old;
1318 }
agg_sum_int32
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32(int32_t *agg, const int32_t val)
Definition: RuntimeFunctions.cpp:1212

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

GPU_RT_STUB int32_t agg_sum_int32_skip_val_shared ( int32_t *  agg,
const int32_t  val,
const int32_t  skip_val 
)

Definition at line 1759 of file RuntimeFunctions.cpp.

1761  {
1762  return 0;
1763 }
GPU_RT_STUB int64_t agg_sum_shared ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1733 of file RuntimeFunctions.cpp.

1733  {
1734  return 0;
1735 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum_skip_val ( int64_t *  agg,
const int64_t  val,
const int64_t  skip_val 
)

Definition at line 1293 of file RuntimeFunctions.cpp.

References agg_sum().

Referenced by agg_sum_if_skip_val(), and Executor::reduceResults().

1295  {
1296  const auto old = *agg;
1297  if (val != skip_val) {
1298  if (old != skip_val) {
1299  return agg_sum(agg, val);
1300  } else {
1301  *agg = val;
1302  }
1303  }
1304  return old;
1305 }
agg_sum
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum(int64_t *agg, const int64_t val)
Definition: RuntimeFunctions.cpp:1130

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

GPU_RT_STUB int64_t agg_sum_skip_val_shared ( int64_t *  agg,
const int64_t  val,
const int64_t  skip_val 
)

Definition at line 1743 of file RuntimeFunctions.cpp.

1745  {
1746  return 0;
1747 }
RUNTIME_EXPORT ALWAYS_INLINE int8_t bit_is_set ( const int8_t *  bitset,
const int64_t  val,
const int64_t  min_val,
const int64_t  max_val,
const int64_t  null_val,
const int8_t  null_bool_val 
)

Definition at line 407 of file RuntimeFunctions.cpp.

412  {
413  if (val == null_val) {
414  return null_bool_val;
415  }
416  if (val < min_val || val > max_val) {
417  return 0;
418  }
419  if (!bitset) {
420  return 0;
421  }
422  const uint64_t bitmap_idx = val - min_val;
423  return bitset[bitmap_idx >> 3] & (1 << (bitmap_idx & 7)) ? 1 : 0;
424 }
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t char_length ( const char *  str,
const int32_t  str_len 
)

Definition at line 2137 of file RuntimeFunctions.cpp.

Referenced by ScalarExprVisitor< std::set< shared::TableKey > >::visit().

2137  {
2138  return str_len;
2139 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t char_length_nullable ( const char *  str,
const int32_t  str_len,
const int32_t  int_null 
)

Definition at line 2142 of file RuntimeFunctions.cpp.

2142  {
2143  if (!str) {
2144  return int_null;
2145  }
2146  return str_len;
2147 }
RUNTIME_EXPORT ALWAYS_INLINE DEVICE bool check_interrupt ( )

Definition at line 2496 of file RuntimeFunctions.cpp.

References check_interrupt_init(), INT_CHECK, and runtime_interrupt_flag.

2496  {
2497  if (check_interrupt_init(static_cast<unsigned>(INT_CHECK))) {
2498  return true;
2499  }
2500  return false;
2501 }
INT_CHECK
Definition: RuntimeFunctions.h:174
check_interrupt_init
RUNTIME_EXPORT bool check_interrupt_init(unsigned command)
Definition: RuntimeFunctions.cpp:2503

+ Here is the call graph for this function:

RUNTIME_EXPORT bool check_interrupt_init ( unsigned  command)

Definition at line 2503 of file RuntimeFunctions.cpp.

References INT_ABORT, INT_CHECK, INT_RESET, and runtime_interrupt_flag.

Referenced by check_interrupt(), Executor::interrupt(), and Executor::resetInterrupt().

2503  {
2504  static std::atomic_bool runtime_interrupt_flag{false};
2505 
2506  if (command == static_cast<unsigned>(INT_CHECK)) {
2507  if (runtime_interrupt_flag.load()) {
2508  return true;
2509  }
2510  return false;
2511  }
2512  if (command == static_cast<unsigned>(INT_ABORT)) {
2513  runtime_interrupt_flag.store(true);
2514  return false;
2515  }
2516  if (command == static_cast<unsigned>(INT_RESET)) {
2517  runtime_interrupt_flag.store(false);
2518  return false;
2519  }
2520  return false;
2521 }
INT_RESET
Definition: RuntimeFunctions.h:174
INT_CHECK
Definition: RuntimeFunctions.h:174
runtime_interrupt_flag
__device__ int32_t runtime_interrupt_flag
Definition: cuda_mapd_rt.cu:95
INT_ABORT
Definition: RuntimeFunctions.h:174

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t checked_single_agg_id ( int64_t *  agg,
const int64_t  val,
const int64_t  null_val 
)

Definition at line 1165 of file RuntimeFunctions.cpp.

1165  {
1166  if (val == null_val) {
1167  return 0;
1168  }
1169 
1170  if (*agg == val) {
1171  return 0;
1172  } else if (*agg == null_val) {
1173  *agg = val;
1174  return 0;
1175  } else {
1176  // see Execute::ERR_SINGLE_VALUE_FOUND_MULTIPLE_VALUES
1177  return 15;
1178  }
1179 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t checked_single_agg_id_double ( int64_t *  agg,
const double  val,
const double  null_val 
)

Definition at line 1456 of file RuntimeFunctions.cpp.

1456  {
1457  if (val == null_val) {
1458  return 0;
1459  }
1460 
1461  if (*agg == *(reinterpret_cast<const int64_t*>(may_alias_ptr(&val)))) {
1462  return 0;
1463  } else if (*agg == *(reinterpret_cast<const int64_t*>(may_alias_ptr(&null_val)))) {
1464  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&val)));
1465  return 0;
1466  } else {
1467  // see Execute::ERR_SINGLE_VALUE_FOUND_MULTIPLE_VALUES
1468  return 15;
1469  }
1470 }
GPU_RT_STUB int32_t checked_single_agg_id_double_shared ( int64_t *  agg,
const double  val,
const double  null_val 
)

Definition at line 1703 of file RuntimeFunctions.cpp.

1705  {
1706  return 0;
1707 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t checked_single_agg_id_float ( int32_t *  agg,
const float  val,
const float  null_val 
)

Definition at line 1508 of file RuntimeFunctions.cpp.

1508  {
1509  if (val == null_val) {
1510  return 0;
1511  }
1512 
1513  if (*agg == *(reinterpret_cast<const int32_t*>(may_alias_ptr(&val)))) {
1514  return 0;
1515  } else if (*agg == *(reinterpret_cast<const int32_t*>(may_alias_ptr(&null_val)))) {
1516  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&val)));
1517  return 0;
1518  } else {
1519  // see Execute::ERR_SINGLE_VALUE_FOUND_MULTIPLE_VALUES
1520  return 15;
1521  }
1522 }
GPU_RT_STUB int32_t checked_single_agg_id_float_shared ( int32_t *  agg,
const float  val,
const float  null_val 
)

Definition at line 1709 of file RuntimeFunctions.cpp.

1711  {
1712  return 0;
1713 }
GPU_RT_STUB int32_t checked_single_agg_id_int16_shared ( int16_t *  agg,
const int16_t  val,
const int16_t  null_val 
)

Definition at line 1691 of file RuntimeFunctions.cpp.

1693  {
1694  return 0;
1695 }
GPU_RT_STUB int32_t checked_single_agg_id_int32_shared ( int32_t *  agg,
const int32_t  val,
const int32_t  null_val 
)

Definition at line 1685 of file RuntimeFunctions.cpp.

1687  {
1688  return 0;
1689 }
GPU_RT_STUB int32_t checked_single_agg_id_int8_shared ( int8_t *  agg,
const int8_t  val,
const int8_t  null_val 
)

Definition at line 1696 of file RuntimeFunctions.cpp.

1698  {
1699  return 0;
1700 }
GPU_RT_STUB int32_t checked_single_agg_id_shared ( int64_t *  agg,
const int64_t  val,
const int64_t  null_val 
)

Definition at line 1678 of file RuntimeFunctions.cpp.

1680  {
1681  return 0;
1682 }
template<typename T , typename Comparator >
int64_t compute_current_row_idx_in_frame ( const int64_t  num_elems,
const int64_t  cur_row_idx,
const T *  col_buf,
const int32_t *  partition_rowid_buf,
const int64_t *  ordered_index_buf,
const T  null_val,
const bool  nulls_first,
const int64_t  null_start_pos,
const int64_t  null_end_pos,
Comparator  cmp 
)
inline

Definition at line 456 of file RuntimeFunctions.cpp.

References get_valid_buf_end_pos(), and get_valid_buf_start_pos().

465  {
466  const auto target_value = col_buf[cur_row_idx];
467  if (target_value == null_val) {
468  for (int64_t target_offset = null_start_pos; target_offset < null_end_pos;
469  target_offset++) {
470  const auto candidate_offset = partition_rowid_buf[ordered_index_buf[target_offset]];
471  if (candidate_offset == cur_row_idx) {
472  return target_offset;
473  }
474  }
475  }
476  auto const modified_null_end_pos = nulls_first ? null_end_pos - 1 : null_end_pos;
477  int64_t l = get_valid_buf_start_pos(null_start_pos, modified_null_end_pos);
478  int64_t h = get_valid_buf_end_pos(num_elems, null_start_pos, modified_null_end_pos);
479  while (l < h) {
480  int64_t mid = l + (h - l) / 2;
481  auto const target_row_idx = partition_rowid_buf[ordered_index_buf[mid]];
482  auto const cur_value = col_buf[target_row_idx];
483  if (cmp(target_value, cur_value)) {
484  h = mid;
485  } else {
486  l = mid + 1;
487  }
488  }
489  int64_t target_offset = l;
490  int64_t candidate_row_idx = partition_rowid_buf[ordered_index_buf[target_offset]];
491  while (col_buf[candidate_row_idx] == target_value && target_offset < num_elems) {
492  if (candidate_row_idx == cur_row_idx) {
493  return target_offset;
494  }
495  candidate_row_idx = partition_rowid_buf[ordered_index_buf[++target_offset]];
496  }
497  return -1;
498 }
get_valid_buf_start_pos
RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_start_pos(const int64_t null_start_pos, const int64_t null_end_pos)
Definition: RuntimeFunctions.cpp:444
get_valid_buf_end_pos
RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_end_pos(const int64_t num_elems, const int64_t null_start_pos, const int64_t null_end_pos)
Definition: RuntimeFunctions.cpp:449

+ Here is the call graph for this function:

template<typename AGG_VALUE_TYPE >
void compute_derived_aggregates ( SumAndCountPair< AGG_VALUE_TYPE > *  aggregation_tree_for_partition,
SumAndCountPair< AGG_VALUE_TYPE > &  res,
size_t  query_range_start_idx,
size_t  query_range_end_idx,
size_t  leaf_level,
size_t  tree_fanout,
AGG_VALUE_TYPE  invalid_val,
AGG_VALUE_TYPE  null_val 
)
inline

Definition at line 958 of file RuntimeFunctions.cpp.

References SumAndCountPair< T >::count, getStartOffsetForSegmentTreeTraversal(), and SumAndCountPair< T >::sum.

966  {
967  size_t leaf_start_idx = getStartOffsetForSegmentTreeTraversal(leaf_level, tree_fanout);
968  size_t begin = leaf_start_idx + query_range_start_idx;
969  size_t end = leaf_start_idx + query_range_end_idx;
970  SumAndCountPair<AGG_VALUE_TYPE> null_res{null_val, 0};
971  SumAndCountPair<AGG_VALUE_TYPE> invalid_res{invalid_val, 0};
972  bool all_nulls = true;
973  for (int level = leaf_level; level >= 0; level--) {
974  size_t parentBegin = begin / tree_fanout;
975  size_t parentEnd = (end - 1) / tree_fanout;
976  if (parentBegin == parentEnd) {
977  for (size_t pos = begin; pos < end; pos++) {
978  if (aggregation_tree_for_partition[pos].sum != null_val) {
979  all_nulls = false;
980  res.sum += aggregation_tree_for_partition[pos].sum;
981  res.count += aggregation_tree_for_partition[pos].count;
982  }
983  }
984  if (all_nulls) {
985  res = null_res;
986  }
987  return;
988  } else if (parentBegin > parentEnd) {
989  res = null_res;
990  return;
991  }
992  size_t group_begin = (parentBegin * tree_fanout) + 1;
993  if (begin != group_begin) {
994  size_t limit = (parentBegin * tree_fanout) + tree_fanout + 1;
995  for (size_t pos = begin; pos < limit; pos++) {
996  if (aggregation_tree_for_partition[pos].sum != null_val) {
997  all_nulls = false;
998  res.sum += aggregation_tree_for_partition[pos].sum;
999  res.count += aggregation_tree_for_partition[pos].count;
1000  }
1001  }
1002  parentBegin++;
1003  }
1004  size_t group_end = (parentEnd * tree_fanout) + 1;
1005  if (end != group_end) {
1006  for (size_t pos = group_end; pos < end; pos++) {
1007  if (aggregation_tree_for_partition[pos].sum != null_val) {
1008  all_nulls = false;
1009  res.sum += aggregation_tree_for_partition[pos].sum;
1010  res.count += aggregation_tree_for_partition[pos].count;
1011  }
1012  }
1013  }
1014  begin = parentBegin;
1015  end = parentEnd;
1016  }
1017  res = invalid_res;
1018  return;
1019 }
getStartOffsetForSegmentTreeTraversal
RUNTIME_EXPORT ALWAYS_INLINE size_t getStartOffsetForSegmentTreeTraversal(size_t level, size_t tree_fanout)
Definition: RuntimeFunctions.cpp:823

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t compute_int64_t_lower_bound ( const int64_t  entry_cnt,
const int64_t  target_value,
const int64_t *  col_buf 
)

Definition at line 427 of file RuntimeFunctions.cpp.

429  {
430  int64_t l = 0;
431  int64_t h = entry_cnt - 1;
432  while (l < h) {
433  int64_t mid = l + (h - l) / 2;
434  if (target_value < col_buf[mid]) {
435  h = mid;
436  } else {
437  l = mid + 1;
438  }
439  }
440  return l;
441 }
template<typename TARGET_VAL_TYPE , typename COL_TYPE , typename NULL_TYPE >
int64_t compute_lower_bound_from_ordered_partition_index ( const int64_t  num_elems,
const TARGET_VAL_TYPE  target_val,
const COL_TYPE *  col_buf,
const int32_t *  partition_rowid_buf,
const int64_t *  ordered_index_buf,
const NULL_TYPE  null_val,
const bool  nulls_first,
const int64_t  null_start_offset,
const int64_t  null_end_offset 
)
inline

Definition at line 538 of file RuntimeFunctions.cpp.

References get_valid_buf_end_pos(), and get_valid_buf_start_pos().

547  {
548  if (target_val == null_val) {
549  return null_start_offset;
550  }
551  auto const modified_null_end_pos = nulls_first ? null_end_offset - 1 : null_end_offset;
552  int64_t l = get_valid_buf_start_pos(null_start_offset, modified_null_end_pos);
553  int64_t h = get_valid_buf_end_pos(num_elems, null_start_offset, modified_null_end_pos);
554  while (l < h) {
555  int64_t mid = l + (h - l) / 2;
556  if (target_val <= col_buf[partition_rowid_buf[ordered_index_buf[mid]]]) {
557  h = mid;
558  } else {
559  l = mid + 1;
560  }
561  }
562  return l;
563 }
get_valid_buf_start_pos
RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_start_pos(const int64_t null_start_pos, const int64_t null_end_pos)
Definition: RuntimeFunctions.cpp:444
get_valid_buf_end_pos
RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_end_pos(const int64_t num_elems, const int64_t null_start_pos, const int64_t null_end_pos)
Definition: RuntimeFunctions.cpp:449

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t compute_row_mode_end_index_add ( int64_t  candidate_index,
int64_t  current_partition_start_offset,
int64_t  frame_bound,
int64_t  num_current_partition_elem 
)

Definition at line 787 of file RuntimeFunctions.cpp.

790  {
791  int64_t index = candidate_index - current_partition_start_offset + frame_bound;
792  return index >= num_current_partition_elem ? num_current_partition_elem : index + 1;
793 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t compute_row_mode_end_index_sub ( int64_t  candidate_index,
int64_t  current_partition_start_offset,
int64_t  frame_bound 
)

Definition at line 779 of file RuntimeFunctions.cpp.

781  {
782  int64_t index = candidate_index - current_partition_start_offset - frame_bound;
783  return index < 0 ? 0 : index + 1;
784 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t compute_row_mode_start_index_add ( int64_t  candidate_index,
int64_t  current_partition_start_offset,
int64_t  frame_bound,
int64_t  num_current_partition_elem 
)

Definition at line 770 of file RuntimeFunctions.cpp.

773  {
774  int64_t index = candidate_index - current_partition_start_offset + frame_bound;
775  return index >= num_current_partition_elem ? num_current_partition_elem : index;
776 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t compute_row_mode_start_index_sub ( int64_t  candidate_index,
int64_t  current_partition_start_offset,
int64_t  frame_bound 
)

Definition at line 762 of file RuntimeFunctions.cpp.

764  {
765  int64_t index = candidate_index - current_partition_start_offset - frame_bound;
766  return index < 0 ? 0 : index;
767 }
template<typename TARGET_VAL_TYPE , typename COL_TYPE , typename NULL_TYPE >
int64_t compute_upper_bound_from_ordered_partition_index ( const int64_t  num_elems,
const TARGET_VAL_TYPE  target_val,
const COL_TYPE *  col_buf,
const int32_t *  partition_rowid_buf,
const int64_t *  ordered_index_buf,
const NULL_TYPE  null_val,
const bool  nulls_first,
const int64_t  null_start_offset,
const int64_t  null_end_offset 
)
inline

Definition at line 619 of file RuntimeFunctions.cpp.

References get_valid_buf_end_pos(), and get_valid_buf_start_pos().

628  {
629  if (target_val == null_val) {
630  return null_end_offset;
631  }
632  auto const modified_null_end_pos = nulls_first ? null_end_offset - 1 : null_end_offset;
633  int64_t l = get_valid_buf_start_pos(null_start_offset, modified_null_end_pos);
634  int64_t h = get_valid_buf_end_pos(num_elems, null_start_offset, modified_null_end_pos);
635  while (l < h) {
636  int64_t mid = l + (h - l) / 2;
637  if (target_val >= col_buf[partition_rowid_buf[ordered_index_buf[mid]]]) {
638  l = mid + 1;
639  } else {
640  h = mid;
641  }
642  }
643  return l;
644 }
get_valid_buf_start_pos
RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_start_pos(const int64_t null_start_pos, const int64_t null_end_pos)
Definition: RuntimeFunctions.cpp:444
get_valid_buf_end_pos
RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_end_pos(const int64_t num_elems, const int64_t null_start_pos, const int64_t null_end_pos)
Definition: RuntimeFunctions.cpp:449

+ Here is the call graph for this function:

template<AggFuncType AGG_FUNC_TYPE, typename AGG_TYPE >
AGG_TYPE compute_window_func_via_aggregation_tree ( AGG_TYPE *  aggregation_tree_for_partition,
size_t  query_range_start_idx,
size_t  query_range_end_idx,
size_t  leaf_level,
size_t  tree_fanout,
AGG_TYPE  init_val,
AGG_TYPE  invalid_val,
AGG_TYPE  null_val 
)
inline

Definition at line 846 of file RuntimeFunctions.cpp.

References getStartOffsetForSegmentTreeTraversal(), and run_benchmark_import::res.

854  {
855  size_t leaf_start_idx = getStartOffsetForSegmentTreeTraversal(leaf_level, tree_fanout);
856  size_t begin = leaf_start_idx + query_range_start_idx;
857  size_t end = leaf_start_idx + query_range_end_idx;
858  AGG_TYPE res = init_val;
859  bool all_nulls = true;
860  for (int level = leaf_level; level >= 0; level--) {
861  size_t parentBegin = begin / tree_fanout;
862  size_t parentEnd = (end - 1) / tree_fanout;
863  if (parentBegin == parentEnd) {
864  for (size_t pos = begin; pos < end; pos++) {
865  if (aggregation_tree_for_partition[pos] != null_val) {
866  all_nulls = false;
867  res = agg_func<AGG_FUNC_TYPE>(res, aggregation_tree_for_partition[pos]);
868  }
869  }
870  return all_nulls ? null_val : res;
871  } else if (parentBegin > parentEnd) {
872  return null_val;
873  }
874  size_t group_begin = (parentBegin * tree_fanout) + 1;
875  if (begin != group_begin) {
876  size_t limit = (parentBegin * tree_fanout) + tree_fanout + 1;
877  for (size_t pos = begin; pos < limit; pos++) {
878  if (aggregation_tree_for_partition[pos] != null_val) {
879  all_nulls = false;
880  res = agg_func<AGG_FUNC_TYPE>(res, aggregation_tree_for_partition[pos]);
881  }
882  }
883  parentBegin++;
884  }
885  size_t group_end = (parentEnd * tree_fanout) + 1;
886  if (end != group_end) {
887  for (size_t pos = group_end; pos < end; pos++) {
888  if (aggregation_tree_for_partition[pos] != null_val) {
889  all_nulls = false;
890  res = agg_func<AGG_FUNC_TYPE>(res, aggregation_tree_for_partition[pos]);
891  }
892  }
893  }
894  begin = parentBegin;
895  end = parentEnd;
896  }
897  return invalid_val;
898 }
run_benchmark_import.res
tuple res
Definition: run_benchmark_import.py:381
getStartOffsetForSegmentTreeTraversal
RUNTIME_EXPORT ALWAYS_INLINE size_t getStartOffsetForSegmentTreeTraversal(size_t level, size_t tree_fanout)
Definition: RuntimeFunctions.cpp:823

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t decimal_ceil ( const int64_t  x,
const int64_t  scale 
)

Definition at line 1593 of file RuntimeFunctions.cpp.

References decimal_floor().

1594  {
1595  return decimal_floor(x, scale) + (x % scale ? scale : 0);
1596 }
decimal_floor
RUNTIME_EXPORT ALWAYS_INLINE int64_t decimal_floor(const int64_t x, const int64_t scale)
Definition: RuntimeFunctions.cpp:1582

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t decimal_floor ( const int64_t  x,
const int64_t  scale 
)

Definition at line 1582 of file RuntimeFunctions.cpp.

Referenced by decimal_ceil().

1583  {
1584  if (x >= 0) {
1585  return x / scale * scale;
1586  }
1587  if (!(x % scale)) {
1588  return x;
1589  }
1590  return x / scale * scale - scale;
1591 }

+ Here is the caller graph for this function:

GPU_RT_STUB int64_t* declare_dynamic_shared_memory ( )

Definition at line 1838 of file RuntimeFunctions.cpp.

1838  {
1839  return nullptr;
1840 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t encode_date ( int64_t  decoded_val,
int64_t  null_val,
int64_t  multiplier 
)

Definition at line 755 of file RuntimeFunctions.cpp.

757  {
758  return decoded_val == null_val ? decoded_val : decoded_val * multiplier;
759 }
template<typename T >
T fill_missing_value ( int64_t const  cur_idx,
T const  null_val,
T *const  col_buf,
int64_t const  num_elems_in_partition,
int32_t *const  partition_rowid_buf,
int64_t *const  ordered_index_buf,
bool const  is_forward_fill 
)

Definition at line 1077 of file RuntimeFunctions.cpp.

References heavydb.dtypes::T.

1083  {
1084  T const cur_val = col_buf[partition_rowid_buf[ordered_index_buf[cur_idx]]];
1085  if (cur_val == null_val) {
1086  if (is_forward_fill) {
1087  for (int64_t cand_idx = cur_idx - 1; cand_idx >= 0; --cand_idx) {
1088  T const candidate_val = col_buf[partition_rowid_buf[ordered_index_buf[cand_idx]]];
1089  if (candidate_val != null_val) {
1090  return candidate_val;
1091  }
1092  }
1093  } else {
1094  for (int64_t cand_idx = cur_idx + 1; cand_idx < num_elems_in_partition;
1095  ++cand_idx) {
1096  T const candidate_val = col_buf[partition_rowid_buf[ordered_index_buf[cand_idx]]];
1097  if (candidate_val != null_val) {
1098  return candidate_val;
1099  }
1100  }
1101  }
1102  }
1103  return cur_val;
1104 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t floor_div_lhs ( const int64_t  dividend,
const int64_t  divisor 
)

Definition at line 232 of file RuntimeFunctions.cpp.

Referenced by floor_div_nullable_lhs().

233  {
234  return (dividend < 0 ? dividend - (divisor - 1) : dividend) / divisor;
235 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t floor_div_nullable_lhs ( const int64_t  dividend,
const int64_t  divisor,
const int64_t  null_val 
)

Definition at line 240 of file RuntimeFunctions.cpp.

References floor_div_lhs().

242  {
243  return dividend == null_val ? null_val : floor_div_lhs(dividend, divisor);
244 }
floor_div_lhs
RUNTIME_EXPORT ALWAYS_INLINE int64_t floor_div_lhs(const int64_t dividend, const int64_t divisor)
Definition: RuntimeFunctions.cpp:232

+ Here is the call graph for this function:

GPU_RT_STUB void force_sync ( )

Definition at line 1806 of file RuntimeFunctions.cpp.

1806 {}
GPU_RT_STUB int64_t get_block_index ( )

Definition at line 1842 of file RuntimeFunctions.cpp.

1842  {
1843  return 0;
1844 }
RUNTIME_EXPORT ALWAYS_INLINE double* get_double_aggregation_tree ( int64_t **  aggregation_trees,
size_t  partition_idx 
)

Definition at line 801 of file RuntimeFunctions.cpp.

803  {
804  double** casted_aggregation_trees = reinterpret_cast<double**>(aggregation_trees);
805  return casted_aggregation_trees[partition_idx];
806 }
RUNTIME_EXPORT ALWAYS_INLINE SumAndCountPair<double>* get_double_derived_aggregation_tree ( int64_t **  aggregation_trees,
size_t  partition_idx 
)

Definition at line 816 of file RuntimeFunctions.cpp.

816  {
817  SumAndCountPair<double>** casted_aggregation_trees =
818  reinterpret_cast<SumAndCountPair<double>**>(aggregation_trees);
819  return casted_aggregation_trees[partition_idx];
820 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t get_error_code ( int32_t *  error_codes)

Definition at line 1861 of file RuntimeFunctions.cpp.

References pos_start_impl().

1861  {
1862  return error_codes[pos_start_impl(nullptr)];
1863 }
pos_start_impl
__device__ int32_t pos_start_impl(const int32_t *row_index_resume)
Definition: cuda_mapd_rt.cu:27

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t* get_group_value_fast_keyless ( int64_t *  groups_buffer,
const int64_t  key,
const int64_t  min_key,
const int64_t  ,
const uint32_t  row_size_quad 
)

Definition at line 2103 of file RuntimeFunctions.cpp.

2108  {
2109  return groups_buffer + row_size_quad * (key - min_key);
2110 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t* get_group_value_fast_keyless_semiprivate ( int64_t *  groups_buffer,
const int64_t  key,
const int64_t  min_key,
const int64_t  ,
const uint32_t  row_size_quad,
const uint8_t  thread_warp_idx,
const uint8_t  warp_size 
)

Definition at line 2112 of file RuntimeFunctions.cpp.

2119  {
2120  return groups_buffer + row_size_quad * (warp_size * (key - min_key) + thread_warp_idx);
2121 }
thread_warp_idx
__device__ int8_t thread_warp_idx(const int8_t warp_sz)
Definition: cuda_mapd_rt.cu:39
RUNTIME_EXPORT ALWAYS_INLINE int64_t* get_integer_aggregation_tree ( int64_t **  aggregation_trees,
size_t  partition_idx 
)

Definition at line 795 of file RuntimeFunctions.cpp.

797  {
798  return aggregation_trees[partition_idx];
799 }
RUNTIME_EXPORT ALWAYS_INLINE SumAndCountPair<int64_t>* get_integer_derived_aggregation_tree ( int64_t **  aggregation_trees,
size_t  partition_idx 
)

Definition at line 809 of file RuntimeFunctions.cpp.

809  {
810  SumAndCountPair<int64_t>** casted_aggregation_trees =
811  reinterpret_cast<SumAndCountPair<int64_t>**>(aggregation_trees);
812  return casted_aggregation_trees[partition_idx];
813 }
template<typename T >
ALWAYS_INLINE int64_t* get_matching_group_value ( int64_t *  groups_buffer,
const uint32_t  h,
const T *  key,
const uint32_t  key_count,
const uint32_t  row_size_quad 
)

Definition at line 1931 of file RuntimeFunctions.cpp.

References align_to_int64(), and heavydb.dtypes::T.

1935  {
1936  auto off = h * row_size_quad;
1937  auto row_ptr = reinterpret_cast<T*>(groups_buffer + off);
1938  if (*row_ptr == get_empty_key<T>()) {
1939  memcpy(row_ptr, key, key_count * sizeof(T));
1940  auto row_ptr_i8 = reinterpret_cast<int8_t*>(row_ptr + key_count);
1941  return reinterpret_cast<int64_t*>(align_to_int64(row_ptr_i8));
1942  }
1943  if (memcmp(row_ptr, key, key_count * sizeof(T)) == 0) {
1944  auto row_ptr_i8 = reinterpret_cast<int8_t*>(row_ptr + key_count);
1945  return reinterpret_cast<int64_t*>(align_to_int64(row_ptr_i8));
1946  }
1947  return nullptr;
1948 }
align_to_int64
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
Definition: BufferCompaction.h:42

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t* get_matching_group_value ( int64_t *  groups_buffer,
const uint32_t  h,
const int64_t *  key,
const uint32_t  key_count,
const uint32_t  key_width,
const uint32_t  row_size_quad 
)

Definition at line 1950 of file RuntimeFunctions.cpp.

References get_matching_group_value().

1956  {
1957  switch (key_width) {
1958  case 4:
1959  return get_matching_group_value(groups_buffer,
1960  h,
1961  reinterpret_cast<const int32_t*>(key),
1962  key_count,
1963  row_size_quad);
1964  case 8:
1965  return get_matching_group_value(groups_buffer, h, key, key_count, row_size_quad);
1966  default:;
1967  }
1968  return nullptr;
1969 }
get_matching_group_value
__device__ int64_t * get_matching_group_value(int64_t *groups_buffer, const uint32_t h, const T *key, const uint32_t key_count, const uint32_t row_size_quad)
Definition: cuda_mapd_rt.cu:174

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t* get_matching_group_value_columnar ( int64_t *  groups_buffer,
const uint32_t  h,
const int64_t *  key,
const uint32_t  key_qw_count,
const size_t  entry_count 
)

Definition at line 2019 of file RuntimeFunctions.cpp.

References EMPTY_KEY_64.

2024  {
2025  auto off = h;
2026  if (groups_buffer[off] == EMPTY_KEY_64) {
2027  for (size_t i = 0; i < key_qw_count; ++i) {
2028  groups_buffer[off] = key[i];
2029  off += entry_count;
2030  }
2031  return &groups_buffer[off];
2032  }
2033  off = h;
2034  for (size_t i = 0; i < key_qw_count; ++i) {
2035  if (groups_buffer[off] != key[i]) {
2036  return nullptr;
2037  }
2038  off += entry_count;
2039  }
2040  return &groups_buffer[off];
2041 }
EMPTY_KEY_64
#define EMPTY_KEY_64
Definition: GpuRtConstants.h:28
template<typename T >
ALWAYS_INLINE int32_t get_matching_group_value_columnar_slot ( int64_t *  groups_buffer,
const uint32_t  entry_count,
const uint32_t  h,
const T *  key,
const uint32_t  key_count 
)

Definition at line 1972 of file RuntimeFunctions.cpp.

References heavydb.dtypes::T.

1976  {
1977  auto off = h;
1978  auto key_buffer = reinterpret_cast<T*>(groups_buffer);
1979  if (key_buffer[off] == get_empty_key<T>()) {
1980  for (size_t i = 0; i < key_count; ++i) {
1981  key_buffer[off] = key[i];
1982  off += entry_count;
1983  }
1984  return h;
1985  }
1986  off = h;
1987  for (size_t i = 0; i < key_count; ++i) {
1988  if (key_buffer[off] != key[i]) {
1989  return -1;
1990  }
1991  off += entry_count;
1992  }
1993  return h;
1994 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t get_matching_group_value_columnar_slot ( int64_t *  groups_buffer,
const uint32_t  entry_count,
const uint32_t  h,
const int64_t *  key,
const uint32_t  key_count,
const uint32_t  key_width 
)

Definition at line 1997 of file RuntimeFunctions.cpp.

References get_matching_group_value_columnar_slot().

2002  {
2003  switch (key_width) {
2004  case 4:
2005  return get_matching_group_value_columnar_slot(groups_buffer,
2006  entry_count,
2007  h,
2008  reinterpret_cast<const int32_t*>(key),
2009  key_count);
2010  case 8:
2011  return get_matching_group_value_columnar_slot(
2012  groups_buffer, entry_count, h, key, key_count);
2013  default:
2014  return -1;
2015  }
2016  return -1;
2017 }
get_matching_group_value_columnar_slot
__device__ int32_t get_matching_group_value_columnar_slot(int64_t *groups_buffer, const uint32_t entry_count, const uint32_t h, const T *key, const uint32_t key_count)
Definition: cuda_mapd_rt.cu:236

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t* get_matching_group_value_perfect_hash ( int64_t *  groups_buffer,
const uint32_t  hashed_index,
const int64_t *  key,
const uint32_t  key_count,
const uint32_t  row_size_quad 
)

Definition at line 2054 of file RuntimeFunctions.cpp.

References EMPTY_KEY_64.

2059  {
2060  uint32_t off = hashed_index * row_size_quad;
2061  if (groups_buffer[off] == EMPTY_KEY_64) {
2062  for (uint32_t i = 0; i < key_count; ++i) {
2063  groups_buffer[off + i] = key[i];
2064  }
2065  }
2066  return groups_buffer + off + key_count;
2067 }
EMPTY_KEY_64
#define EMPTY_KEY_64
Definition: GpuRtConstants.h:28
RUNTIME_EXPORT ALWAYS_INLINE int64_t* get_matching_group_value_perfect_hash_keyless ( int64_t *  groups_buffer,
const uint32_t  hashed_index,
const uint32_t  row_size_quad 
)

For a particular hashed index (only used with multi-column perfect hash group by) it returns the row-wise offset of the group in the output buffer. Since it is intended for keyless hash use, it assumes there is no group columns prepending the output buffer.

Definition at line 2076 of file RuntimeFunctions.cpp.

2078  {
2079  return groups_buffer + row_size_quad * hashed_index;
2080 }
GPU_RT_STUB int64_t get_thread_index ( )

Definition at line 1834 of file RuntimeFunctions.cpp.

1834  {
1835  return 0;
1836 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_end_pos ( const int64_t  num_elems,
const int64_t  null_start_pos,
const int64_t  null_end_pos 
)

Definition at line 449 of file RuntimeFunctions.cpp.

Referenced by compute_current_row_idx_in_frame(), compute_lower_bound_from_ordered_partition_index(), and compute_upper_bound_from_ordered_partition_index().

451  {
452  return null_end_pos == num_elems ? null_start_pos : num_elems;
453 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t get_valid_buf_start_pos ( const int64_t  null_start_pos,
const int64_t  null_end_pos 
)

Definition at line 444 of file RuntimeFunctions.cpp.

Referenced by compute_current_row_idx_in_frame(), compute_lower_bound_from_ordered_partition_index(), and compute_upper_bound_from_ordered_partition_index().

444  {
445  return null_start_pos == 0 ? null_end_pos + 1 : 0;
446 }

+ Here is the caller graph for this function:

template<typename COL_TYPE , typename LOGICAL_TYPE >
LOGICAL_TYPE get_value_in_window_frame ( const int64_t  target_row_idx_in_frame,
const int64_t  frame_start_offset,
const int64_t  frame_end_offset,
const COL_TYPE *  col_buf,
const int32_t *  partition_rowid_buf,
const int64_t *  ordered_index_buf,
const LOGICAL_TYPE  logical_null_val,
const LOGICAL_TYPE  col_null_val 
)
inline

Definition at line 700 of file RuntimeFunctions.cpp.

707  {
708  if (target_row_idx_in_frame < frame_start_offset ||
709  target_row_idx_in_frame > frame_end_offset) {
710  return logical_null_val;
711  }
712  const auto target_offset =
713  partition_rowid_buf[ordered_index_buf[target_row_idx_in_frame]];
714  LOGICAL_TYPE target_val = col_buf[target_offset];
715  if (target_val == col_null_val) {
716  return logical_null_val;
717  }
718  return target_val;
719 }
RUNTIME_EXPORT ALWAYS_INLINE size_t getStartOffsetForSegmentTreeTraversal ( size_t  level,
size_t  tree_fanout 
)

Definition at line 823 of file RuntimeFunctions.cpp.

Referenced by compute_derived_aggregates(), and compute_window_func_via_aggregation_tree().

823  {
824  size_t offset = 0;
825  for (size_t i = 0; i < level; i++) {
826  offset += pow(tree_fanout, i);
827  }
828  return offset;
829 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT NEVER_INLINE int32_t group_buff_idx_impl ( )

Definition at line 1822 of file RuntimeFunctions.cpp.

References pos_start_impl().

1822  {
1823  return pos_start_impl(nullptr);
1824 }
pos_start_impl
__device__ int32_t pos_start_impl(const int32_t *row_index_resume)
Definition: cuda_mapd_rt.cu:27

+ Here is the call graph for this function:

RUNTIME_EXPORT NEVER_INLINE void init_columnar_group_by_buffer_gpu ( int64_t *  groups_buffer,
const int64_t *  init_vals,
const uint32_t  groups_buffer_entry_count,
const uint32_t  key_qw_count,
const uint32_t  agg_col_count,
const bool  keyless,
const bool  blocks_share_memory,
const int32_t  frag_idx 
)

Definition at line 1901 of file RuntimeFunctions.cpp.

1909  {
1910 #ifndef _WIN32
1911  // the body is not really needed, just make sure the call is not optimized away
1912  assert(groups_buffer);
1913 #endif
1914 }
RUNTIME_EXPORT NEVER_INLINE void init_group_by_buffer_gpu ( int64_t *  groups_buffer,
const int64_t *  init_vals,
const uint32_t  groups_buffer_entry_count,
const uint32_t  key_qw_count,
const uint32_t  agg_col_count,
const bool  keyless,
const int8_t  warp_size 
)

Definition at line 1887 of file RuntimeFunctions.cpp.

1894  {
1895 #ifndef _WIN32
1896  // the body is not really needed, just make sure the call is not optimized away
1897  assert(groups_buffer);
1898 #endif
1899 }
RUNTIME_EXPORT NEVER_INLINE void init_group_by_buffer_impl ( int64_t *  groups_buffer,
const int64_t *  init_vals,
const uint32_t  groups_buffer_entry_count,
const uint32_t  key_qw_count,
const uint32_t  agg_col_count,
const bool  keyless,
const int8_t  warp_size 
)

Definition at line 1916 of file RuntimeFunctions.cpp.

1923  {
1924 #ifndef _WIN32
1925  // the body is not really needed, just make sure the call is not optimized away
1926  assert(groups_buffer);
1927 #endif
1928 }
RUNTIME_EXPORT int64_t* init_shared_mem ( const int64_t *  global_groups_buffer,
const int32_t  groups_buffer_size 
)

Definition at line 1882 of file RuntimeFunctions.cpp.

1883  {
1884  return nullptr;
1885 }
RUNTIME_EXPORT NEVER_INLINE const int64_t* init_shared_mem_nop ( const int64_t *  groups_buffer,
const int32_t  groups_buffer_size 
)

Definition at line 1867 of file RuntimeFunctions.cpp.

1869  {
1870  return groups_buffer;
1871 }
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t key_for_string_encoded ( const int32_t  str_id)

Definition at line 2150 of file RuntimeFunctions.cpp.

2150  {
2151  return str_id;
2152 }
RUNTIME_EXPORT NEVER_INLINE void linear_probabilistic_count ( uint8_t *  bitmap,
const uint32_t  bitmap_bytes,
const uint8_t *  key_bytes,
const uint32_t  key_len 
)

Definition at line 2374 of file RuntimeFunctions.cpp.

References MurmurHash3().

2378  {
2379  const uint32_t bit_pos = MurmurHash3(key_bytes, key_len, 0) % (bitmap_bytes * 8);
2380  const uint32_t word_idx = bit_pos / 32;
2381  const uint32_t bit_idx = bit_pos % 32;
2382  reinterpret_cast<uint32_t*>(bitmap)[word_idx] |= 1 << bit_idx;
2383 }
MurmurHash3
RUNTIME_EXPORT NEVER_INLINE DEVICE uint32_t MurmurHash3(const void *key, int len, const uint32_t seed)
Definition: MurmurHash.cpp:33

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE double load_avg_decimal ( const int64_t *  sum,
const int64_t *  count,
const double  null_val,
const uint32_t  scale 
)

Definition at line 2353 of file RuntimeFunctions.cpp.

2356  {
2357  return *count != 0 ? (static_cast<double>(*sum) / pow(10, scale)) / *count : null_val;
2358 }
RUNTIME_EXPORT ALWAYS_INLINE double load_avg_double ( const int64_t *  agg,
const int64_t *  count,
const double  null_val 
)

Definition at line 2360 of file RuntimeFunctions.cpp.

2362  {
2363  return *count != 0 ? *reinterpret_cast<const double*>(may_alias_ptr(agg)) / *count
2364  : null_val;
2365 }
RUNTIME_EXPORT ALWAYS_INLINE double load_avg_float ( const int32_t *  agg,
const int32_t *  count,
const double  null_val 
)

Definition at line 2367 of file RuntimeFunctions.cpp.

2369  {
2370  return *count != 0 ? *reinterpret_cast<const float*>(may_alias_ptr(agg)) / *count
2371  : null_val;
2372 }
RUNTIME_EXPORT ALWAYS_INLINE double load_avg_int ( const int64_t *  sum,
const int64_t *  count,
const double  null_val 
)

Definition at line 2347 of file RuntimeFunctions.cpp.

2349  {
2350  return *count != 0 ? static_cast<double>(*sum) / *count : null_val;
2351 }
RUNTIME_EXPORT ALWAYS_INLINE double load_double ( const int64_t *  agg)

Definition at line 2339 of file RuntimeFunctions.cpp.

2339  {
2340  return *reinterpret_cast<const double*>(may_alias_ptr(agg));
2341 }
RUNTIME_EXPORT ALWAYS_INLINE float load_float ( const int32_t *  agg)

Definition at line 2343 of file RuntimeFunctions.cpp.

2343  {
2344  return *reinterpret_cast<const float*>(may_alias_ptr(agg));
2345 }
RUNTIME_EXPORT ALWAYS_INLINE int8_t logical_and ( const int8_t  lhs,
const int8_t  rhs,
const int8_t  null_val 
)

Definition at line 335 of file RuntimeFunctions.cpp.

Referenced by process_detections().

337  {
338  if (lhs == null_val) {
339  return rhs == 0 ? rhs : null_val;
340  }
341  if (rhs == null_val) {
342  return lhs == 0 ? lhs : null_val;
343  }
344  return (lhs && rhs) ? 1 : 0;
345 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int8_t logical_not ( const int8_t  operand,
const int8_t  null_val 
)

Definition at line 330 of file RuntimeFunctions.cpp.

331  {
332  return operand == null_val ? operand : (operand ? 0 : 1);
333 }
RUNTIME_EXPORT ALWAYS_INLINE int8_t logical_or ( const int8_t  lhs,
const int8_t  rhs,
const int8_t  null_val 
)

Definition at line 347 of file RuntimeFunctions.cpp.

349  {
350  if (lhs == null_val) {
351  return rhs == 0 ? null_val : rhs;
352  }
353  if (rhs == null_val) {
354  return lhs == 0 ? null_val : lhs;
355  }
356  return (lhs || rhs) ? 1 : 0;
357 }
ALWAYS_INLINE DEVICE int32_t map_string_dict_id ( const int32_t  string_id,
const int64_t  translation_map_handle,
const int32_t  min_source_id 
)

Definition at line 2155 of file RuntimeFunctions.cpp.

2157  {
2158  const int32_t* translation_map =
2159  reinterpret_cast<const int32_t*>(translation_map_handle);
2160  return translation_map[string_id - min_source_id];
2161 }
RUNTIME_EXPORT void multifrag_query ( int32_t *  error_code,
int32_t *  total_matched,
int64_t **  out,
const uint32_t *  num_fragments_ptr,
const uint32_t *  num_tables_ptr,
const uint32_t *  row_index_resume,
const int8_t ***  col_buffers,
const int64_t *  num_rows,
const uint64_t *  frag_row_offsets,
const int32_t *  max_matched,
const int64_t *  init_agg_value,
const int64_t *  join_hash_tables,
const int8_t *  row_func_mgr 
)

Definition at line 2464 of file RuntimeFunctions.cpp.

References query_stub().

2476  {
2477  uint32_t const num_fragments = *num_fragments_ptr;
2478  uint32_t const num_tables = *num_tables_ptr;
2479  // num_fragments_ptr and num_tables_ptr are replaced by frag_idx when passed below.
2480  for (uint32_t frag_idx = 0; frag_idx < num_fragments; ++frag_idx) {
2481  query_stub(error_code,
2482  total_matched,
2483  out,
2484  frag_idx,
2485  row_index_resume,
2486  col_buffers ? col_buffers[frag_idx] : nullptr,
2487  &num_rows[frag_idx * num_tables],
2488  &frag_row_offsets[frag_idx * num_tables],
2489  max_matched,
2490  init_agg_value,
2491  join_hash_tables,
2492  row_func_mgr);
2493  }
2494 }
query_stub
RUNTIME_EXPORT NEVER_INLINE void query_stub(int32_t *error_code, int32_t *total_matched, int64_t **out, const uint32_t frag_idx, const uint32_t *row_index_resume, const int8_t **col_buffers, const int64_t *num_rows, const uint64_t *frag_row_offsets, const int32_t *max_matched, const int64_t *init_agg_value, const int64_t *join_hash_tables, const int8_t *row_func_mgr)
Definition: RuntimeFunctions.cpp:2444
report.error_code
def error_code
Definition: report.py:244

+ Here is the call graph for this function:

RUNTIME_EXPORT void multifrag_query_hoisted_literals ( int32_t *  error_code,
int32_t *  total_matched,
int64_t **  out,
const uint32_t *  num_fragments_ptr,
const uint32_t *  num_tables_ptr,
const uint32_t *  row_index_resume,
const int8_t ***  col_buffers,
const int8_t *  literals,
const int64_t *  num_rows,
const uint64_t *  frag_row_offsets,
const int32_t *  max_matched,
const int64_t *  init_agg_value,
const int64_t *  join_hash_tables,
const int8_t *  row_func_mgr 
)

Definition at line 2408 of file RuntimeFunctions.cpp.

References query_stub_hoisted_literals().

2422  {
2423  uint32_t const num_fragments = *num_fragments_ptr;
2424  uint32_t const num_tables = *num_tables_ptr;
2425  // num_fragments_ptr and num_tables_ptr are replaced by frag_idx when passed below.
2426  for (uint32_t frag_idx = 0; frag_idx < num_fragments; ++frag_idx) {
2427  query_stub_hoisted_literals(error_code,
2428  total_matched,
2429  out,
2430  frag_idx,
2431  row_index_resume,
2432  col_buffers ? col_buffers[frag_idx] : nullptr,
2433  literals,
2434  &num_rows[frag_idx * num_tables],
2435  &frag_row_offsets[frag_idx * num_tables],
2436  max_matched,
2437  init_agg_value,
2438  join_hash_tables,
2439  row_func_mgr);
2440  }
2441 }
query_stub_hoisted_literals
RUNTIME_EXPORT NEVER_INLINE void query_stub_hoisted_literals(int32_t *error_code, int32_t *total_matched, int64_t **out, const uint32_t frag_idx, const uint32_t *row_index_resume, const int8_t **col_buffers, const int8_t *literals, const int64_t *num_rows, const uint64_t *frag_row_offsets, const int32_t *max_matched, const int64_t *init_agg_value, const int64_t *join_hash_tables, const int8_t *row_func_mgr)
Definition: RuntimeFunctions.cpp:2386
report.error_code
def error_code
Definition: report.py:244

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE double percent_window_func ( const int64_t  output_buff,
const int64_t  pos 
)

Definition at line 2333 of file RuntimeFunctions.cpp.

2335  {
2336  return reinterpret_cast<const double*>(output_buff)[pos];
2337 }
RUNTIME_EXPORT NEVER_INLINE int32_t pos_start_impl ( int32_t const *  row_index_resume)

Definition at line 1818 of file RuntimeFunctions.cpp.

1818  {
1819  return row_index_resume ? *row_index_resume : 0;
1820 }
RUNTIME_EXPORT NEVER_INLINE int32_t pos_step_impl ( )

Definition at line 1826 of file RuntimeFunctions.cpp.

1826  {
1827  return 1;
1828 }
RUNTIME_EXPORT NEVER_INLINE void query_stub ( int32_t *  error_code,
int32_t *  total_matched,
int64_t **  out,
const uint32_t  frag_idx,
const uint32_t *  row_index_resume,
const int8_t **  col_buffers,
const int64_t *  num_rows,
const uint64_t *  frag_row_offsets,
const int32_t *  max_matched,
const int64_t *  init_agg_value,
const int64_t *  join_hash_tables,
const int8_t *  row_func_mgr 
)

Definition at line 2444 of file RuntimeFunctions.cpp.

Referenced by multifrag_query().

2455  {
2456 #ifndef _WIN32
2457  assert(error_code || total_matched || out || frag_idx || row_index_resume ||
2458  col_buffers || num_rows || frag_row_offsets || max_matched || init_agg_value ||
2459  join_hash_tables || row_func_mgr);
2460 #endif
2461 }
report.error_code
def error_code
Definition: report.py:244

+ Here is the caller graph for this function:

RUNTIME_EXPORT NEVER_INLINE void query_stub_hoisted_literals ( int32_t *  error_code,
int32_t *  total_matched,
int64_t **  out,
const uint32_t  frag_idx,
const uint32_t *  row_index_resume,
const int8_t **  col_buffers,
const int8_t *  literals,
const int64_t *  num_rows,
const uint64_t *  frag_row_offsets,
const int32_t *  max_matched,
const int64_t *  init_agg_value,
const int64_t *  join_hash_tables,
const int8_t *  row_func_mgr 
)

Definition at line 2386 of file RuntimeFunctions.cpp.

Referenced by multifrag_query_hoisted_literals().

2399  {
2400 #ifndef _WIN32
2401  assert(error_code || total_matched || out || frag_idx || row_index_resume ||
2402  col_buffers || literals || num_rows || frag_row_offsets || max_matched ||
2403  init_agg_value || join_hash_tables || row_func_mgr);
2404 #endif
2405 }
report.error_code
def error_code
Definition: report.py:244

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE void record_error_code ( const int32_t  err_code,
int32_t *  error_codes 
)

Definition at line 1848 of file RuntimeFunctions.cpp.

References pos_start_impl().

1849  {
1850  // NB: never override persistent error codes (with code greater than zero).
1851  // On GPU, a projection query with a limit can run out of slots without it
1852  // being an actual error if the limit has been hit. If a persistent error
1853  // (division by zero, for example) occurs before running out of slots, we
1854  // have to avoid overriding it, because there's a risk that the query would
1855  // go through if we override with a potentially benign out-of-slots code.
1856  if (err_code && error_codes[pos_start_impl(nullptr)] <= 0) {
1857  error_codes[pos_start_impl(nullptr)] = err_code;
1858  }
1859 }
pos_start_impl
__device__ int32_t pos_start_impl(const int32_t *row_index_resume)
Definition: cuda_mapd_rt.cu:27

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t row_number_window_func ( const int64_t  output_buff,
const int64_t  pos 
)

Definition at line 2329 of file RuntimeFunctions.cpp.

2329  {
2330  return reinterpret_cast<const int64_t*>(output_buff)[pos];
2331 }
RUNTIME_EXPORT ALWAYS_INLINE DEVICE bool sample_ratio ( const double  proportion,
const int64_t  row_offset 
)

Definition at line 2198 of file RuntimeFunctions.cpp.

Referenced by ScalarExprVisitor< std::set< shared::TableKey > >::visit().

2200  {
2201  const int64_t threshold = 4294967296 * proportion;
2202  return (row_offset * 2654435761) % 4294967296 < threshold;
2203 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int64_t scale_decimal_down_not_nullable ( const int64_t  operand,
const int64_t  scale,
const int64_t  null_val 
)

Definition at line 222 of file RuntimeFunctions.cpp.

224  {
225  int64_t tmp = scale >> 1;
226  tmp = operand >= 0 ? operand + tmp : operand - tmp;
227  return tmp / scale;
228 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t scale_decimal_down_nullable ( const int64_t  operand,
const int64_t  scale,
const int64_t  null_val 
)

Definition at line 208 of file RuntimeFunctions.cpp.

210  {
211  // rounded scale down of a decimal
212  if (operand == null_val) {
213  return null_val;
214  }
215 
216  int64_t tmp = scale >> 1;
217  tmp = operand >= 0 ? operand + tmp : operand - tmp;
218  return tmp / scale;
219 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t scale_decimal_up ( const int64_t  operand,
const uint64_t  scale,
const int64_t  operand_null_val,
const int64_t  result_null_val 
)

Definition at line 200 of file RuntimeFunctions.cpp.

203  {
204  return operand != operand_null_val ? operand * scale : result_null_val;
205 }
RUNTIME_EXPORT ALWAYS_INLINE void set_matching_group_value_perfect_hash_columnar ( int64_t *  groups_buffer,
const uint32_t  hashed_index,
const int64_t *  key,
const uint32_t  key_count,
const uint32_t  entry_count 
)

Definition at line 2087 of file RuntimeFunctions.cpp.

References EMPTY_KEY_64.

2091  {
2092  if (groups_buffer[hashed_index] == EMPTY_KEY_64) {
2093  for (uint32_t i = 0; i < key_count; i++) {
2094  groups_buffer[i * entry_count + hashed_index] = key[i];
2095  }
2096  }
2097 }
EMPTY_KEY_64
#define EMPTY_KEY_64
Definition: GpuRtConstants.h:28
RUNTIME_EXPORT ALWAYS_INLINE StringView string_pack ( const int8_t *  ptr,
const int32_t  len 
)

Definition at line 2123 of file RuntimeFunctions.cpp.

2124  {
2125  return {reinterpret_cast<char const*>(ptr), static_cast<uint64_t>(len)};
2126 }
GPU_RT_STUB void sync_threadblock ( )

Definition at line 1810 of file RuntimeFunctions.cpp.

1810 {}
GPU_RT_STUB void sync_warp ( )

Definition at line 1808 of file RuntimeFunctions.cpp.

1808 {}
GPU_RT_STUB void sync_warp_protected ( int64_t  thread_pos,
int64_t  row_count 
)

Definition at line 1809 of file RuntimeFunctions.cpp.

1809 {}
GPU_RT_STUB int8_t thread_warp_idx ( const int8_t  warp_sz)

Definition at line 1830 of file RuntimeFunctions.cpp.

1830  {
1831  return 0;
1832 }
ALWAYS_INLINE DEVICE double tree_model_reg_predict ( const double *  regressor_inputs,
const int64_t  decision_tree_table_handle,
const int64_t  decision_tree_offsets_handle,
const int32_t  num_regressors,
const int32_t  num_trees,
const bool  compute_avg,
const double  null_value 
)

Definition at line 2163 of file RuntimeFunctions.cpp.

References DecisionTreeEntry::feature_index, DecisionTreeEntry::isSplitNode(), DecisionTreeEntry::left_child_row_idx, DecisionTreeEntry::right_child_row_idx, and DecisionTreeEntry::value.

2170  {
2171  for (int32_t regressor_idx = 0; regressor_idx < num_regressors; ++regressor_idx) {
2172  if (regressor_inputs[regressor_idx] == null_value) {
2173  return null_value;
2174  }
2175  }
2176  const DecisionTreeEntry* decision_tree_table =
2177  reinterpret_cast<const DecisionTreeEntry*>(decision_tree_table_handle);
2178  const int64_t* decision_tree_offsets =
2179  reinterpret_cast<const int64_t*>(decision_tree_offsets_handle);
2180  double sum_tree_results{0};
2181  for (int32_t tree_idx = 0; tree_idx < num_trees; ++tree_idx) {
2182  int64_t row_idx = decision_tree_offsets[tree_idx];
2183  while (true) {
2184  const DecisionTreeEntry& current_entry = decision_tree_table[row_idx];
2185  if (!current_entry.isSplitNode()) {
2186  sum_tree_results += current_entry.value;
2187  break;
2188  }
2189  const auto regressor_input = regressor_inputs[current_entry.feature_index];
2190  row_idx = regressor_input <= current_entry.value
2191  ? current_entry.left_child_row_idx
2192  : current_entry.right_child_row_idx;
2193  }
2194  }
2195  return compute_avg ? sum_tree_results / num_trees : sum_tree_results;
2196 }
DecisionTreeEntry::value
double value
Definition: DecisionTreeEntry.h:20
DecisionTreeEntry::left_child_row_idx
int64_t left_child_row_idx
Definition: DecisionTreeEntry.h:22
DecisionTreeEntry::feature_index
int64_t feature_index
Definition: DecisionTreeEntry.h:21
DecisionTreeEntry
Definition: DecisionTreeEntry.h:19
DecisionTreeEntry::isSplitNode
bool isSplitNode() const
Definition: DecisionTreeEntry.h:39
DecisionTreeEntry::right_child_row_idx
int64_t right_child_row_idx
Definition: DecisionTreeEntry.h:23

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket ( const double  target_value,
const double  lower_bound,
const double  upper_bound,
const double  scale_factor,
const int32_t  partition_count 
)

Definition at line 2206 of file RuntimeFunctions.cpp.

Referenced by ScalarExprVisitor< std::set< shared::TableKey > >::visit(), width_bucket_expr(), and width_bucket_nullable().

2210  {
2211  if (target_value < lower_bound) {
2212  return 0;
2213  } else if (target_value >= upper_bound) {
2214  return partition_count + 1;
2215  }
2216  return ((target_value - lower_bound) * scale_factor) + 1;
2217 }
gpu_enabled::upper_bound
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr ( const double  target_value,
const bool  reversed,
const double  lower_bound,
const double  upper_bound,
const int32_t  partition_count 
)

Definition at line 2281 of file RuntimeFunctions.cpp.

References width_bucket(), and width_bucket_reversed().

Referenced by CodeGenerator::codegen(), getExpressionRange(), and width_bucket_expr_nullable().

2285  {
2286  if (reversed) {
2287  return width_bucket_reversed(target_value,
2288  lower_bound,
2289  upper_bound,
2290  partition_count / (lower_bound - upper_bound),
2291  partition_count);
2292  }
2293  return width_bucket(target_value,
2294  lower_bound,
2295  upper_bound,
2296  partition_count / (upper_bound - lower_bound),
2297  partition_count);
2298 }
gpu_enabled::upper_bound
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
width_bucket
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket(const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count)
Definition: RuntimeFunctions.cpp:2206
width_bucket_reversed
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_reversed(const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count)
Definition: RuntimeFunctions.cpp:2220
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr_no_oob_check ( const double  target_value,
const bool  reversed,
const double  lower_bound,
const double  upper_bound,
const int32_t  partition_count 
)

Definition at line 2315 of file RuntimeFunctions.cpp.

References width_bucket_no_oob_check(), and width_bucket_reversed_no_oob_check().

2319  {
2320  if (reversed) {
2321  return width_bucket_reversed_no_oob_check(
2322  target_value, lower_bound, partition_count / (lower_bound - upper_bound));
2323  }
2324  return width_bucket_no_oob_check(
2325  target_value, lower_bound, partition_count / (upper_bound - lower_bound));
2326 }
gpu_enabled::upper_bound
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
width_bucket_no_oob_check
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_no_oob_check(const double target_value, const double lower_bound, const double scale_factor)
Definition: RuntimeFunctions.cpp:2265
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78
width_bucket_reversed_no_oob_check
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_reversed_no_oob_check(const double target_value, const double lower_bound, const double scale_factor)
Definition: RuntimeFunctions.cpp:2273

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr_nullable ( const double  target_value,
const bool  reversed,
const double  lower_bound,
const double  upper_bound,
const int32_t  partition_count,
const double  null_val 
)

Definition at line 2301 of file RuntimeFunctions.cpp.

References width_bucket_expr().

2306  {
2307  if (target_value == null_val) {
2308  return INT32_MIN;
2309  }
2310  return width_bucket_expr(
2311  target_value, reversed, lower_bound, upper_bound, partition_count);
2312 }
gpu_enabled::upper_bound
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78
width_bucket_expr
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_expr(const double target_value, const bool reversed, const double lower_bound, const double upper_bound, const int32_t partition_count)
Definition: RuntimeFunctions.cpp:2281

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_no_oob_check ( const double  target_value,
const double  lower_bound,
const double  scale_factor 
)

Definition at line 2265 of file RuntimeFunctions.cpp.

References gpu_enabled::lower_bound().

Referenced by width_bucket_expr_no_oob_check().

2267  {
2268  int32_t calc = (target_value - lower_bound) * scale_factor;
2269  return calc + 1;
2270 }
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t width_bucket_nullable ( const double  target_value,
const double  lower_bound,
const double  upper_bound,
const double  scale_factor,
const int32_t  partition_count,
const double  null_val 
)

Definition at line 2234 of file RuntimeFunctions.cpp.

References width_bucket().

2239  {
2240  if (target_value == null_val) {
2241  return INT32_MIN;
2242  }
2243  return width_bucket(
2244  target_value, lower_bound, upper_bound, scale_factor, partition_count);
2245 }
gpu_enabled::upper_bound
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
width_bucket
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket(const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count)
Definition: RuntimeFunctions.cpp:2206
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78

+ Here is the call graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_reversed ( const double  target_value,
const double  lower_bound,
const double  upper_bound,
const double  scale_factor,
const int32_t  partition_count 
)

Definition at line 2220 of file RuntimeFunctions.cpp.

Referenced by width_bucket_expr(), and width_bucket_reversed_nullable().

2224  {
2225  if (target_value > lower_bound) {
2226  return 0;
2227  } else if (target_value <= upper_bound) {
2228  return partition_count + 1;
2229  }
2230  return ((lower_bound - target_value) * scale_factor) + 1;
2231 }
gpu_enabled::upper_bound
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_reversed_no_oob_check ( const double  target_value,
const double  lower_bound,
const double  scale_factor 
)

Definition at line 2273 of file RuntimeFunctions.cpp.

Referenced by width_bucket_expr_no_oob_check().

2275  {
2276  int32_t calc = (lower_bound - target_value) * scale_factor;
2277  return calc + 1;
2278 }
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78

+ Here is the caller graph for this function:

RUNTIME_EXPORT ALWAYS_INLINE int32_t width_bucket_reversed_nullable ( const double  target_value,
const double  lower_bound,
const double  upper_bound,
const double  scale_factor,
const int32_t  partition_count,
const double  null_val 
)

Definition at line 2248 of file RuntimeFunctions.cpp.

References width_bucket_reversed().

2253  {
2254  if (target_value == null_val) {
2255  return INT32_MIN;
2256  }
2257  return width_bucket_reversed(
2258  target_value, lower_bound, upper_bound, scale_factor, partition_count);
2259 }
gpu_enabled::upper_bound
DEVICE auto upper_bound(ARGS &&...args)
Definition: gpu_enabled.h:123
width_bucket_reversed
RUNTIME_EXPORT ALWAYS_INLINE DEVICE int32_t width_bucket_reversed(const double target_value, const double lower_bound, const double upper_bound, const double scale_factor, const int32_t partition_count)
Definition: RuntimeFunctions.cpp:2220
gpu_enabled::lower_bound
DEVICE auto lower_bound(ARGS &&...args)
Definition: gpu_enabled.h:78

+ Here is the call graph for this function:

GPU_RT_STUB void write_back_non_grouped_agg ( int64_t *  input_buffer,
int64_t *  output_buffer,
const int32_t  num_agg_cols 
)

Definition at line 1812 of file RuntimeFunctions.cpp.

1814  {};
RUNTIME_EXPORT NEVER_INLINE void write_back_nop ( int64_t *  dest,
int64_t *  src,
const int32_t  sz 
)

Definition at line 1873 of file RuntimeFunctions.cpp.

1875  {
1876 #ifndef _WIN32
1877  // the body is not really needed, just make sure the call is not optimized away
1878  assert(dest);
1879 #endif
1880 }
run_benchmark_import.dest
string dest
Definition: run_benchmark_import.py:74