OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
RuntimeFunctions.h File Reference
#include "Shared/funcannotations.h"
#include <cassert>
#include <cstdint>
#include <ctime>
#include <limits>
#include <type_traits>
+ Include dependency graph for RuntimeFunctions.h:
+ This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define EMPTY_KEY_64   std::numeric_limits<int64_t>::max()
 
#define EMPTY_KEY_32   std::numeric_limits<int32_t>::max()
 
#define EMPTY_KEY_16   std::numeric_limits<int16_t>::max()
 
#define EMPTY_KEY_8   std::numeric_limits<int8_t>::max()
 

Enumerations

enum  RuntimeInterruptFlags { INT_CHECK = 0, INT_ABORT = -1, INT_RESET = -2 }
 

Functions

RUNTIME_EXPORT int64_t agg_sum (int64_t *agg, const int64_t val)
 
RUNTIME_EXPORT void agg_max (int64_t *agg, const int64_t val)
 
RUNTIME_EXPORT void agg_min (int64_t *agg, const int64_t val)
 
RUNTIME_EXPORT void agg_sum_double (int64_t *agg, const double val)
 
RUNTIME_EXPORT void agg_max_double (int64_t *agg, const double val)
 
RUNTIME_EXPORT void agg_min_double (int64_t *agg, const double val)
 
RUNTIME_EXPORT int32_t agg_sum_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val)
 
RUNTIME_EXPORT int64_t agg_sum_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val)
 
RUNTIME_EXPORT void agg_max_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val)
 
RUNTIME_EXPORT void agg_min_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val)
 
RUNTIME_EXPORT void agg_sum_float_skip_val (int32_t *agg, const float val, const float skip_val)
 
RUNTIME_EXPORT void agg_sum_double_skip_val (int64_t *agg, const double val, const double skip_val)
 
RUNTIME_EXPORT void agg_max_double_skip_val (int64_t *agg, const double val, const double skip_val)
 
RUNTIME_EXPORT void agg_min_double_skip_val (int64_t *agg, const double val, const double skip_val)
 
RUNTIME_EXPORT int32_t agg_sum_int32 (int32_t *agg, const int32_t val)
 
RUNTIME_EXPORT void agg_max_int32 (int32_t *agg, const int32_t val)
 
RUNTIME_EXPORT void agg_max_int16 (int16_t *agg, const int16_t val)
 
RUNTIME_EXPORT void agg_max_int8 (int8_t *agg, const int8_t val)
 
RUNTIME_EXPORT void agg_min_int32 (int32_t *agg, const int32_t val)
 
RUNTIME_EXPORT void agg_min_int16 (int16_t *agg, const int16_t val)
 
RUNTIME_EXPORT void agg_min_int8 (int8_t *agg, const int8_t val)
 
RUNTIME_EXPORT void agg_sum_float (int32_t *agg, const float val)
 
RUNTIME_EXPORT void agg_max_float (int32_t *agg, const float val)
 
RUNTIME_EXPORT void agg_min_float (int32_t *agg, const float val)
 
RUNTIME_EXPORT void agg_max_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val)
 
RUNTIME_EXPORT void agg_max_int16_skip_val (int16_t *agg, const int16_t val, const int16_t skip_val)
 
RUNTIME_EXPORT void agg_max_int8_skip_val (int8_t *agg, const int8_t val, const int8_t skip_val)
 
RUNTIME_EXPORT void agg_min_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val)
 
RUNTIME_EXPORT void agg_min_int16_skip_val (int16_t *agg, const int16_t val, const int16_t skip_val)
 
RUNTIME_EXPORT void agg_min_int8_skip_val (int8_t *agg, const int8_t val, const int8_t skip_val)
 
RUNTIME_EXPORT void agg_max_float_skip_val (int32_t *agg, const float val, const float skip_val)
 
RUNTIME_EXPORT void agg_min_float_skip_val (int32_t *agg, const float val, const float skip_val)
 
RUNTIME_EXPORT void agg_count_distinct_bitmap (int64_t *agg, const int64_t val, const int64_t min_val)
 
RUNTIME_EXPORT uint32_t key_hash (const int64_t *key, const uint32_t key_qw_count, const uint32_t key_byte_width)
 
RUNTIME_EXPORT int64_t * get_group_value (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad)
 
bool RUNTIME_EXPORT check_interrupt ()
 
bool RUNTIME_EXPORT check_interrupt_init (unsigned command)
 
RUNTIME_EXPORT int64_t * get_group_value_with_watchdog (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad)
 
RUNTIME_EXPORT int64_t * get_group_value_columnar (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_qw_count)
 
RUNTIME_EXPORT int64_t * get_group_value_columnar_with_watchdog (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_qw_count)
 
RUNTIME_EXPORT int64_t * get_group_value_fast (int64_t *groups_buffer, const int64_t key, const int64_t min_key, const int64_t bucket, const uint32_t row_size_quad)
 
RUNTIME_EXPORT int64_t * get_group_value_fast_with_original_key (int64_t *groups_buffer, const int64_t key, const int64_t orig_key, const int64_t min_key, const int64_t bucket, const uint32_t row_size_quad)
 
RUNTIME_EXPORT uint32_t get_columnar_group_bin_offset (int64_t *key_base_ptr, const int64_t key, const int64_t min_key, const int64_t bucket)
 
RUNTIME_EXPORT int64_t * get_matching_group_value_perfect_hash (int64_t *groups_buffer, const uint32_t h, const int64_t *key, const uint32_t key_qw_count, const uint32_t row_size_quad)
 
RUNTIME_EXPORT int64_t * get_matching_group_value_perfect_hash_keyless (int64_t *groups_buffer, const uint32_t hashed_index, const uint32_t row_size_quad)
 
RUNTIME_EXPORT int32_t * get_bucketized_hash_slot (int32_t *buff, const int64_t key, const int64_t min_key, const int64_t bucket_normalization=1)
 
RUNTIME_EXPORT int32_t * get_hash_slot (int32_t *buff, const int64_t key, const int64_t min_key)
 
RUNTIME_EXPORT int32_t * get_hash_slot_sharded (int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t num_shards, const uint32_t device_count)
 
RUNTIME_EXPORT int32_t * get_bucketized_hash_slot_sharded (int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t num_shards, const uint32_t device_count, const int64_t bucket_normalization)
 
RUNTIME_EXPORT int32_t * get_hash_slot_sharded_opt (int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t shard, const uint32_t num_shards, const uint32_t device_count)
 
RUNTIME_EXPORT int32_t * get_bucketized_hash_slot_sharded_opt (int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t shard, const uint32_t num_shards, const uint32_t device_count, const int64_t bucket_normalization)
 
RUNTIME_EXPORT int fill_one_to_one_hashtable (size_t idx, int32_t *entry_ptr, const int32_t invalid_slot_val)
 
RUNTIME_EXPORT int fill_hashtable_for_semi_join (size_t idx, int32_t *entry_ptr, const int32_t invalid_slot_val)
 
RUNTIME_EXPORT void linear_probabilistic_count (uint8_t *bitmap, const uint32_t bitmap_bytes, const uint8_t *key_bytes, const uint32_t key_len)
 
RUNTIME_EXPORT int64_t fixed_width_int_decode_noinline (const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
 
RUNTIME_EXPORT int64_t fixed_width_unsigned_decode_noinline (const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
 
RUNTIME_EXPORT float fixed_width_float_decode_noinline (const int8_t *byte_stream, const int64_t pos)
 
RUNTIME_EXPORT double fixed_width_double_decode_noinline (const int8_t *byte_stream, const int64_t pos)
 
RUNTIME_EXPORT int64_t fixed_width_small_date_decode_noinline (const int8_t *byte_stream, const int32_t byte_width, const int32_t null_val, const int64_t ret_null_val, const int64_t pos)
 
RUNTIME_EXPORT int8_t * extract_str_ptr_noinline (const uint64_t str_and_len)
 
RUNTIME_EXPORT int32_t extract_str_len_noinline (const uint64_t str_and_len)
 
template<typename T = int64_t>
get_empty_key ()
 
template<>
int32_t get_empty_key ()
 

Macro Definition Documentation

#define EMPTY_KEY_16   std::numeric_limits<int16_t>::max()

Definition at line 122 of file RuntimeFunctions.h.

#define EMPTY_KEY_32   std::numeric_limits<int32_t>::max()

Definition at line 121 of file RuntimeFunctions.h.

Referenced by get_empty_key().

#define EMPTY_KEY_64   std::numeric_limits<int64_t>::max()

Definition at line 120 of file RuntimeFunctions.h.

Referenced by get_empty_key().

#define EMPTY_KEY_8   std::numeric_limits<int8_t>::max()

Definition at line 123 of file RuntimeFunctions.h.

Enumeration Type Documentation

Enumerator
INT_CHECK 
INT_ABORT 
INT_RESET 

Definition at line 137 of file RuntimeFunctions.h.

Function Documentation

RUNTIME_EXPORT void agg_count_distinct_bitmap ( int64_t *  agg,
const int64_t  val,
const int64_t  min_val 
)

Definition at line 345 of file RuntimeFunctions.cpp.

Referenced by agg_count_distinct_bitmap_skip_val(), WindowFunctionContext::fillPartitionEnd(), WindowFunctionContext::fillPartitionStart(), anonymous_namespace{WindowContext.cpp}::index_to_partition_end(), and InValuesBitmap::InValuesBitmap().

345  {
346  const uint64_t bitmap_idx = val - min_val;
347  reinterpret_cast<int8_t*>(*agg)[bitmap_idx >> 3] |= (1 << (bitmap_idx & 7));
348 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_max ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1031 of file RuntimeFunctions.cpp.

1031  {
1032  *agg = std::max(*agg, val);
1033 }
RUNTIME_EXPORT void agg_max_double ( int64_t *  agg,
const double  val 
)

Definition at line 1269 of file RuntimeFunctions.cpp.

1270  {
1271  const auto r = std::max(*reinterpret_cast<const double*>(agg), val);
1272  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&r)));
1273 }
RUNTIME_EXPORT void agg_max_double_skip_val ( int64_t *  agg,
const double  val,
const double  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_max_float ( int32_t *  agg,
const float  val 
)

Definition at line 1314 of file RuntimeFunctions.cpp.

1315  {
1316  const auto r = std::max(*reinterpret_cast<const float*>(agg), val);
1317  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&r)));
1318 }
RUNTIME_EXPORT void agg_max_float_skip_val ( int32_t *  agg,
const float  val,
const float  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_max_int16 ( int16_t *  agg,
const int16_t  val 
)
RUNTIME_EXPORT void agg_max_int16_skip_val ( int16_t *  agg,
const int16_t  val,
const int16_t  skip_val 
)
RUNTIME_EXPORT void agg_max_int32 ( int32_t *  agg,
const int32_t  val 
)
RUNTIME_EXPORT void agg_max_int32_skip_val ( int32_t *  agg,
const int32_t  val,
const int32_t  skip_val 
)
RUNTIME_EXPORT void agg_max_int8 ( int8_t *  agg,
const int8_t  val 
)
RUNTIME_EXPORT void agg_max_int8_skip_val ( int8_t *  agg,
const int8_t  val,
const int8_t  skip_val 
)
RUNTIME_EXPORT void agg_max_skip_val ( int64_t *  agg,
const int64_t  val,
const int64_t  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_min ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1035 of file RuntimeFunctions.cpp.

1035  {
1036  *agg = std::min(*agg, val);
1037 }
RUNTIME_EXPORT void agg_min_double ( int64_t *  agg,
const double  val 
)

Definition at line 1275 of file RuntimeFunctions.cpp.

1276  {
1277  const auto r = std::min(*reinterpret_cast<const double*>(agg), val);
1278  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&r)));
1279 }
RUNTIME_EXPORT void agg_min_double_skip_val ( int64_t *  agg,
const double  val,
const double  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_min_float ( int32_t *  agg,
const float  val 
)

Definition at line 1320 of file RuntimeFunctions.cpp.

1321  {
1322  const auto r = std::min(*reinterpret_cast<const float*>(agg), val);
1323  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&r)));
1324 }
RUNTIME_EXPORT void agg_min_float_skip_val ( int32_t *  agg,
const float  val,
const float  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_min_int16 ( int16_t *  agg,
const int16_t  val 
)
RUNTIME_EXPORT void agg_min_int16_skip_val ( int16_t *  agg,
const int16_t  val,
const int16_t  skip_val 
)
RUNTIME_EXPORT void agg_min_int32 ( int32_t *  agg,
const int32_t  val 
)
RUNTIME_EXPORT void agg_min_int32_skip_val ( int32_t *  agg,
const int32_t  val,
const int32_t  skip_val 
)
RUNTIME_EXPORT void agg_min_int8 ( int8_t *  agg,
const int8_t  val 
)
RUNTIME_EXPORT void agg_min_int8_skip_val ( int8_t *  agg,
const int8_t  val,
const int8_t  skip_val 
)
RUNTIME_EXPORT void agg_min_skip_val ( int64_t *  agg,
const int64_t  val,
const int64_t  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT int64_t agg_sum ( int64_t *  agg,
const int64_t  val 
)

Definition at line 1025 of file RuntimeFunctions.cpp.

Referenced by agg_sum_skip_val().

1025  {
1026  const auto old = *agg;
1027  *agg += val;
1028  return old;
1029 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_sum_double ( int64_t *  agg,
const double  val 
)

Definition at line 1263 of file RuntimeFunctions.cpp.

1264  {
1265  const auto r = *reinterpret_cast<const double*>(agg) + val;
1266  *agg = *reinterpret_cast<const int64_t*>(may_alias_ptr(&r));
1267 }
RUNTIME_EXPORT void agg_sum_double_skip_val ( int64_t *  agg,
const double  val,
const double  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT void agg_sum_float ( int32_t *  agg,
const float  val 
)

Definition at line 1308 of file RuntimeFunctions.cpp.

1309  {
1310  const auto r = *reinterpret_cast<const float*>(agg) + val;
1311  *agg = *reinterpret_cast<const int32_t*>(may_alias_ptr(&r));
1312 }
RUNTIME_EXPORT void agg_sum_float_skip_val ( int32_t *  agg,
const float  val,
const float  skip_val 
)

Referenced by Executor::reduceResults().

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t agg_sum_int32 ( int32_t *  agg,
const int32_t  val 
)

Definition at line 1094 of file RuntimeFunctions.cpp.

Referenced by agg_sum_int32_skip_val().

1095  {
1096  const auto old = *agg;
1097  *agg += val;
1098  return old;
1099 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t agg_sum_int32_skip_val ( int32_t *  agg,
const int32_t  val,
const int32_t  skip_val 
)

Definition at line 1184 of file RuntimeFunctions.cpp.

References agg_sum_int32().

1184  {
1185  const auto old = *agg;
1186  if (val != skip_val) {
1187  if (old != skip_val) {
1188  return agg_sum_int32(agg, val);
1189  } else {
1190  *agg = val;
1191  }
1192  }
1193  return old;
1194 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t agg_sum_int32(int32_t *agg, const int32_t val)

+ Here is the call graph for this function:

RUNTIME_EXPORT int64_t agg_sum_skip_val ( int64_t *  agg,
const int64_t  val,
const int64_t  skip_val 
)

Definition at line 1169 of file RuntimeFunctions.cpp.

References agg_sum().

Referenced by Executor::reduceResults().

1171  {
1172  const auto old = *agg;
1173  if (val != skip_val) {
1174  if (old != skip_val) {
1175  return agg_sum(agg, val);
1176  } else {
1177  *agg = val;
1178  }
1179  }
1180  return old;
1181 }
RUNTIME_EXPORT ALWAYS_INLINE int64_t agg_sum(int64_t *agg, const int64_t val)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool RUNTIME_EXPORT check_interrupt ( )

Definition at line 159 of file cuda_mapd_rt.cu.

References check_interrupt_init(), INT_CHECK, and runtime_interrupt_flag.

Referenced by check_interrupt_rt(), and ColumnFetcher::linearizeFixedLenArrayColFrags().

159  {
160  return (runtime_interrupt_flag == 1) ? true : false;
161 }
__device__ int32_t runtime_interrupt_flag
Definition: cuda_mapd_rt.cu:95

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool RUNTIME_EXPORT check_interrupt_init ( unsigned  command)

Definition at line 2245 of file RuntimeFunctions.cpp.

References INT_ABORT, INT_CHECK, INT_RESET, and runtime_interrupt_flag.

Referenced by check_interrupt(), Executor::interrupt(), and Executor::resetInterrupt().

2245  {
2246  static std::atomic_bool runtime_interrupt_flag{false};
2247 
2248  if (command == static_cast<unsigned>(INT_CHECK)) {
2249  if (runtime_interrupt_flag.load()) {
2250  return true;
2251  }
2252  return false;
2253  }
2254  if (command == static_cast<unsigned>(INT_ABORT)) {
2255  runtime_interrupt_flag.store(true);
2256  return false;
2257  }
2258  if (command == static_cast<unsigned>(INT_RESET)) {
2259  runtime_interrupt_flag.store(false);
2260  return false;
2261  }
2262  return false;
2263 }
__device__ int32_t runtime_interrupt_flag
Definition: cuda_mapd_rt.cu:95

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t extract_str_len_noinline ( const uint64_t  str_and_len)

Definition at line 1920 of file RuntimeFunctions.cpp.

References extract_str_len().

Referenced by string_compress().

1920  {
1921  return extract_str_len(str_and_len);
1922 }
RUNTIME_EXPORT ALWAYS_INLINE int32_t extract_str_len(const uint64_t str_and_len)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int8_t* extract_str_ptr_noinline ( const uint64_t  str_and_len)

Definition at line 1914 of file RuntimeFunctions.cpp.

References extract_str_ptr().

Referenced by string_compress().

1915  {
1916  return extract_str_ptr(str_and_len);
1917 }
RUNTIME_EXPORT ALWAYS_INLINE int8_t * extract_str_ptr(const uint64_t str_and_len)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int fill_hashtable_for_semi_join ( size_t  idx,
int32_t *  entry_ptr,
const int32_t  invalid_slot_val 
)

Definition at line 54 of file JoinHashImpl.h.

References insert_key_cas.

Referenced by fill_hash_join_buff(), fill_hash_join_buff_bucketized(), fill_hash_join_buff_sharded(), and fill_hash_join_buff_sharded_bucketized().

57  {
58  // just mark the existence of value to the corresponding hash slot
59  // regardless of hashtable collision
60  insert_key_cas(entry_ptr, invalid_slot_val, idx);
61  return 0;
62 }
#define insert_key_cas(address, compare, val)
Definition: JoinHashImpl.h:40

+ Here is the caller graph for this function:

RUNTIME_EXPORT int fill_one_to_one_hashtable ( size_t  idx,
int32_t *  entry_ptr,
const int32_t  invalid_slot_val 
)

Definition at line 44 of file JoinHashImpl.h.

References insert_key_cas.

Referenced by fill_hash_join_buff(), fill_hash_join_buff_bucketized(), fill_hash_join_buff_sharded(), and fill_hash_join_buff_sharded_bucketized().

47  {
48  if (insert_key_cas(entry_ptr, invalid_slot_val, idx) != invalid_slot_val) {
49  return -1;
50  }
51  return 0;
52 }
#define insert_key_cas(address, compare, val)
Definition: JoinHashImpl.h:40

+ Here is the caller graph for this function:

RUNTIME_EXPORT double fixed_width_double_decode_noinline ( const int8_t *  byte_stream,
const int64_t  pos 
)

Definition at line 133 of file DecodersImpl.h.

References fixed_width_double_decode(), and SUFFIX.

Referenced by compute_bucket_sizes_impl(), JoinColumnIterator::getElementSwitch(), result_set::lazy_decode(), OverlapsKeyHandler::operator()(), and RangeKeyHandler::operator()().

133  {
134  return SUFFIX(fixed_width_double_decode)(byte_stream, pos);
135 }
#define SUFFIX(name)
DEVICE ALWAYS_INLINE double SUFFIX() fixed_width_double_decode(const int8_t *byte_stream, const int64_t pos)
Definition: DecodersImpl.h:125

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT float fixed_width_float_decode_noinline ( const int8_t *  byte_stream,
const int64_t  pos 
)

Definition at line 120 of file DecodersImpl.h.

References fixed_width_float_decode(), and SUFFIX.

Referenced by result_set::lazy_decode().

120  {
121  return SUFFIX(fixed_width_float_decode)(byte_stream, pos);
122 }
#define SUFFIX(name)
DEVICE ALWAYS_INLINE float SUFFIX() fixed_width_float_decode(const int8_t *byte_stream, const int64_t pos)
Definition: DecodersImpl.h:112

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int64_t fixed_width_int_decode_noinline ( const int8_t *  byte_stream,
const int32_t  byte_width,
const int64_t  pos 
)

Definition at line 90 of file DecodersImpl.h.

References fixed_width_int_decode(), and SUFFIX.

Referenced by JoinColumnIterator::getElementSwitch(), result_set::lazy_decode(), and RangeKeyHandler::operator()().

92  {
93  return SUFFIX(fixed_width_int_decode)(byte_stream, byte_width, pos);
94 }
DEVICE ALWAYS_INLINE int64_t SUFFIX() fixed_width_int_decode(const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
Definition: DecodersImpl.h:30
#define SUFFIX(name)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int64_t fixed_width_small_date_decode_noinline ( const int8_t *  byte_stream,
const int32_t  byte_width,
const int32_t  null_val,
const int64_t  ret_null_val,
const int64_t  pos 
)

Definition at line 148 of file DecodersImpl.h.

References fixed_width_small_date_decode(), and SUFFIX.

Referenced by JoinColumnIterator::getElementSwitch(), and result_set::lazy_decode().

152  {
154  byte_stream, byte_width, null_val, ret_null_val, pos);
155 }
#define SUFFIX(name)
DEVICE ALWAYS_INLINE int64_t SUFFIX() fixed_width_small_date_decode(const int8_t *byte_stream, const int32_t byte_width, const int32_t null_val, const int64_t ret_null_val, const int64_t pos)
Definition: DecodersImpl.h:138

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int64_t fixed_width_unsigned_decode_noinline ( const int8_t *  byte_stream,
const int32_t  byte_width,
const int64_t  pos 
)

Definition at line 97 of file DecodersImpl.h.

References fixed_width_unsigned_decode(), and SUFFIX.

Referenced by JoinColumnIterator::getElementSwitch(), and result_set::lazy_decode().

99  {
100  return SUFFIX(fixed_width_unsigned_decode)(byte_stream, byte_width, pos);
101 }
#define SUFFIX(name)
DEVICE ALWAYS_INLINE int64_t SUFFIX() fixed_width_unsigned_decode(const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
Definition: DecodersImpl.h:60

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t* get_bucketized_hash_slot ( int32_t *  buff,
const int64_t  key,
const int64_t  min_key,
const int64_t  bucket_normalization = 1 
)

Definition at line 66 of file JoinHashImpl.h.

Referenced by bucketized_hash_join_idx(), count_matches_bucketized(), fill_hash_join_buff_bucketized(), and fill_row_ids_bucketized().

70  {
71  return buff + (key - min_key) / bucket_normalization;
72 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t* get_bucketized_hash_slot_sharded ( int32_t *  buff,
const int64_t  key,
const int64_t  min_key,
const uint32_t  entry_count_per_shard,
const uint32_t  num_shards,
const uint32_t  device_count,
const int64_t  bucket_normalization 
)

Definition at line 80 of file JoinHashImpl.h.

References SHARD_FOR_KEY.

Referenced by fill_row_ids_sharded_bucketized().

87  {
88  const uint32_t shard = SHARD_FOR_KEY(key, num_shards);
89  const uint32_t shard_buffer_index =
90  shard / device_count; // shard sub-buffer index within `buff`
91  int32_t* shard_buffer = buff + shard_buffer_index * entry_count_per_shard;
92  return shard_buffer + (key - min_key) / bucket_normalization / num_shards;
93 }
#define SHARD_FOR_KEY(key, num_shards)
Definition: shard_key.h:20

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t* get_bucketized_hash_slot_sharded_opt ( int32_t *  buff,
const int64_t  key,
const int64_t  min_key,
const uint32_t  entry_count_per_shard,
const uint32_t  shard,
const uint32_t  num_shards,
const uint32_t  device_count,
const int64_t  bucket_normalization 
)

Definition at line 109 of file JoinHashImpl.h.

Referenced by fill_hash_join_buff_sharded_bucketized().

117  {
118  const uint32_t shard_buffer_index =
119  shard / device_count; // shard sub-buffer index within `buff`
120  int32_t* shard_buffer = buff + shard_buffer_index * entry_count_per_shard;
121  return shard_buffer + (key - min_key) / bucket_normalization / num_shards;
122 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT uint32_t get_columnar_group_bin_offset ( int64_t *  key_base_ptr,
const int64_t  key,
const int64_t  min_key,
const int64_t  bucket 
)

Definition at line 228 of file GroupByRuntime.cpp.

References EMPTY_KEY_64.

231  {
232  int64_t off = key - min_key;
233  if (bucket) {
234  off /= bucket;
235  }
236  if (key_base_ptr[off] == EMPTY_KEY_64) {
237  key_base_ptr[off] = key;
238  }
239  return off;
240 }
#define EMPTY_KEY_64
template<typename T = int64_t>
T get_empty_key ( )
inline

Definition at line 287 of file RuntimeFunctions.h.

References EMPTY_KEY_64.

287  {
288  static_assert(std::is_same<T, int64_t>::value,
289  "Unsupported template parameter other than int64_t for now");
290  return EMPTY_KEY_64;
291 }
#define EMPTY_KEY_64
template<>
int32_t get_empty_key ( )
inline

Definition at line 294 of file RuntimeFunctions.h.

References EMPTY_KEY_32.

294  {
295  return EMPTY_KEY_32;
296 }
#define EMPTY_KEY_32
RUNTIME_EXPORT int64_t* get_group_value ( int64_t *  groups_buffer,
const uint32_t  groups_buffer_entry_count,
const int64_t *  key,
const uint32_t  key_count,
const uint32_t  key_width,
const uint32_t  row_size_quad 
)

Definition at line 25 of file GroupByRuntime.cpp.

References get_matching_group_value(), and key_hash().

Referenced by ResultSetStorage::moveOneEntryToBuffer().

31  {
32  uint32_t h = key_hash(key, key_count, key_width) % groups_buffer_entry_count;
33  int64_t* matching_group = get_matching_group_value(
34  groups_buffer, h, key, key_count, key_width, row_size_quad);
35  if (matching_group) {
36  return matching_group;
37  }
38  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
39  while (h_probe != h) {
40  matching_group = get_matching_group_value(
41  groups_buffer, h_probe, key, key_count, key_width, row_size_quad);
42  if (matching_group) {
43  return matching_group;
44  }
45  h_probe = (h_probe + 1) % groups_buffer_entry_count;
46  }
47  return NULL;
48 }
__device__ int64_t * get_matching_group_value(int64_t *groups_buffer, const uint32_t h, const T *key, const uint32_t key_count, const uint32_t row_size_quad)
RUNTIME_EXPORT ALWAYS_INLINE DEVICE uint32_t key_hash(const int64_t *key, const uint32_t key_count, const uint32_t key_byte_width)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int64_t* get_group_value_columnar ( int64_t *  groups_buffer,
const uint32_t  groups_buffer_entry_count,
const int64_t *  key,
const uint32_t  key_qw_count 
)

Definition at line 139 of file GroupByRuntime.cpp.

References get_matching_group_value_columnar(), and key_hash().

Referenced by ResultSetStorage::moveOneEntryToBuffer().

143  {
144  uint32_t h = key_hash(key, key_qw_count, sizeof(int64_t)) % groups_buffer_entry_count;
145  int64_t* matching_group = get_matching_group_value_columnar(
146  groups_buffer, h, key, key_qw_count, groups_buffer_entry_count);
147  if (matching_group) {
148  return matching_group;
149  }
150  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
151  while (h_probe != h) {
152  matching_group = get_matching_group_value_columnar(
153  groups_buffer, h_probe, key, key_qw_count, groups_buffer_entry_count);
154  if (matching_group) {
155  return matching_group;
156  }
157  h_probe = (h_probe + 1) % groups_buffer_entry_count;
158  }
159  return NULL;
160 }
__device__ int64_t * get_matching_group_value_columnar(int64_t *groups_buffer, const uint32_t h, const int64_t *key, const uint32_t key_qw_count, const size_t entry_count)
RUNTIME_EXPORT ALWAYS_INLINE DEVICE uint32_t key_hash(const int64_t *key, const uint32_t key_count, const uint32_t key_byte_width)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT int64_t* get_group_value_columnar_with_watchdog ( int64_t *  groups_buffer,
const uint32_t  groups_buffer_entry_count,
const int64_t *  key,
const uint32_t  key_qw_count 
)

Definition at line 163 of file GroupByRuntime.cpp.

References dynamic_watchdog(), get_matching_group_value_columnar(), and key_hash().

166  {
167  uint32_t h = key_hash(key, key_qw_count, sizeof(int64_t)) % groups_buffer_entry_count;
168  int64_t* matching_group = get_matching_group_value_columnar(
169  groups_buffer, h, key, key_qw_count, groups_buffer_entry_count);
170  if (matching_group) {
171  return matching_group;
172  }
173  uint32_t watchdog_countdown = 100;
174  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
175  while (h_probe != h) {
176  matching_group = get_matching_group_value_columnar(
177  groups_buffer, h_probe, key, key_qw_count, groups_buffer_entry_count);
178  if (matching_group) {
179  return matching_group;
180  }
181  h_probe = (h_probe + 1) % groups_buffer_entry_count;
182  if (--watchdog_countdown == 0) {
183  if (dynamic_watchdog()) {
184  return NULL;
185  }
186  watchdog_countdown = 100;
187  }
188  }
189  return NULL;
190 }
__device__ bool dynamic_watchdog()
__device__ int64_t * get_matching_group_value_columnar(int64_t *groups_buffer, const uint32_t h, const int64_t *key, const uint32_t key_qw_count, const size_t entry_count)
RUNTIME_EXPORT ALWAYS_INLINE DEVICE uint32_t key_hash(const int64_t *key, const uint32_t key_count, const uint32_t key_byte_width)

+ Here is the call graph for this function:

RUNTIME_EXPORT int64_t* get_group_value_fast ( int64_t *  groups_buffer,
const int64_t  key,
const int64_t  min_key,
const int64_t  bucket,
const uint32_t  row_size_quad 
)

Definition at line 192 of file GroupByRuntime.cpp.

References EMPTY_KEY_64.

197  {
198  int64_t key_diff = key - min_key;
199  if (bucket) {
200  key_diff /= bucket;
201  }
202  int64_t off = key_diff * row_size_quad;
203  if (groups_buffer[off] == EMPTY_KEY_64) {
204  groups_buffer[off] = key;
205  }
206  return groups_buffer + off + 1;
207 }
#define EMPTY_KEY_64
RUNTIME_EXPORT int64_t* get_group_value_fast_with_original_key ( int64_t *  groups_buffer,
const int64_t  key,
const int64_t  orig_key,
const int64_t  min_key,
const int64_t  bucket,
const uint32_t  row_size_quad 
)

Definition at line 210 of file GroupByRuntime.cpp.

References EMPTY_KEY_64.

215  {
216  int64_t key_diff = key - min_key;
217  if (bucket) {
218  key_diff /= bucket;
219  }
220  int64_t off = key_diff * row_size_quad;
221  if (groups_buffer[off] == EMPTY_KEY_64) {
222  groups_buffer[off] = orig_key;
223  }
224  return groups_buffer + off + 1;
225 }
#define EMPTY_KEY_64
RUNTIME_EXPORT int64_t* get_group_value_with_watchdog ( int64_t *  groups_buffer,
const uint32_t  groups_buffer_entry_count,
const int64_t *  key,
const uint32_t  key_count,
const uint32_t  key_width,
const uint32_t  row_size_quad 
)

Definition at line 52 of file GroupByRuntime.cpp.

References dynamic_watchdog(), get_matching_group_value(), and key_hash().

58  {
59  uint32_t h = key_hash(key, key_count, key_width) % groups_buffer_entry_count;
60  int64_t* matching_group = get_matching_group_value(
61  groups_buffer, h, key, key_count, key_width, row_size_quad);
62  if (matching_group) {
63  return matching_group;
64  }
65  uint32_t watchdog_countdown = 100;
66  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
67  while (h_probe != h) {
68  matching_group = get_matching_group_value(
69  groups_buffer, h_probe, key, key_count, key_width, row_size_quad);
70  if (matching_group) {
71  return matching_group;
72  }
73  h_probe = (h_probe + 1) % groups_buffer_entry_count;
74  if (--watchdog_countdown == 0) {
75  if (dynamic_watchdog()) {
76  return NULL;
77  }
78  watchdog_countdown = 100;
79  }
80  }
81  return NULL;
82 }
__device__ bool dynamic_watchdog()
__device__ int64_t * get_matching_group_value(int64_t *groups_buffer, const uint32_t h, const T *key, const uint32_t key_count, const uint32_t row_size_quad)
RUNTIME_EXPORT ALWAYS_INLINE DEVICE uint32_t key_hash(const int64_t *key, const uint32_t key_count, const uint32_t key_byte_width)

+ Here is the call graph for this function:

RUNTIME_EXPORT int32_t* get_hash_slot ( int32_t *  buff,
const int64_t  key,
const int64_t  min_key 
)

Definition at line 74 of file JoinHashImpl.h.

Referenced by count_matches(), fill_hash_join_buff(), fill_row_ids(), and hash_join_idx().

76  {
77  return buff + (key - min_key);
78 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t* get_hash_slot_sharded ( int32_t *  buff,
const int64_t  key,
const int64_t  min_key,
const uint32_t  entry_count_per_shard,
const uint32_t  num_shards,
const uint32_t  device_count 
)

Definition at line 95 of file JoinHashImpl.h.

References SHARD_FOR_KEY.

Referenced by count_matches_sharded(), fill_row_ids_sharded(), and hash_join_idx_sharded().

101  {
102  const uint32_t shard = SHARD_FOR_KEY(key, num_shards);
103  const uint32_t shard_buffer_index =
104  shard / device_count; // shard sub-buffer index within `buff`
105  int32_t* shard_buffer = buff + shard_buffer_index * entry_count_per_shard;
106  return shard_buffer + (key - min_key) / num_shards;
107 }
#define SHARD_FOR_KEY(key, num_shards)
Definition: shard_key.h:20

+ Here is the caller graph for this function:

RUNTIME_EXPORT int32_t* get_hash_slot_sharded_opt ( int32_t *  buff,
const int64_t  key,
const int64_t  min_key,
const uint32_t  entry_count_per_shard,
const uint32_t  shard,
const uint32_t  num_shards,
const uint32_t  device_count 
)

Definition at line 124 of file JoinHashImpl.h.

Referenced by fill_hash_join_buff_sharded().

131  {
132  const uint32_t shard_buffer_index =
133  shard / device_count; // shard sub-buffer index within `buff`
134  int32_t* shard_buffer = buff + shard_buffer_index * entry_count_per_shard;
135  return shard_buffer + (key - min_key) / num_shards;
136 }

+ Here is the caller graph for this function:

RUNTIME_EXPORT int64_t* get_matching_group_value_perfect_hash ( int64_t *  groups_buffer,
const uint32_t  h,
const int64_t *  key,
const uint32_t  key_qw_count,
const uint32_t  row_size_quad 
)

Definition at line 1835 of file RuntimeFunctions.cpp.

References EMPTY_KEY_64.

1840  {
1841  uint32_t off = hashed_index * row_size_quad;
1842  if (groups_buffer[off] == EMPTY_KEY_64) {
1843  for (uint32_t i = 0; i < key_count; ++i) {
1844  groups_buffer[off + i] = key[i];
1845  }
1846  }
1847  return groups_buffer + off + key_count;
1848 }
#define EMPTY_KEY_64
RUNTIME_EXPORT int64_t* get_matching_group_value_perfect_hash_keyless ( int64_t *  groups_buffer,
const uint32_t  hashed_index,
const uint32_t  row_size_quad 
)

For a particular hashed index (only used with multi-column perfect hash group by) it returns the row-wise offset of the group in the output buffer. Since it is intended for keyless hash use, it assumes there is no group columns prepending the output buffer.

Definition at line 1857 of file RuntimeFunctions.cpp.

1859  {
1860  return groups_buffer + row_size_quad * hashed_index;
1861 }
RUNTIME_EXPORT uint32_t key_hash ( const int64_t *  key,
const uint32_t  key_qw_count,
const uint32_t  key_byte_width 
)

Definition at line 21 of file GroupByRuntime.cpp.

References MurmurHash3().

Referenced by get_group_value(), get_group_value_columnar(), anonymous_namespace{ResultSetReduction.cpp}::get_group_value_columnar_reduction(), get_group_value_columnar_slot(), get_group_value_columnar_slot_with_watchdog(), get_group_value_columnar_with_watchdog(), result_set::get_group_value_reduction(), and get_group_value_with_watchdog().

21  {
22  return MurmurHash3(key, key_byte_width * key_count, 0);
23 }
RUNTIME_EXPORT NEVER_INLINE DEVICE uint32_t MurmurHash3(const void *key, int len, const uint32_t seed)
Definition: MurmurHash.cpp:33

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RUNTIME_EXPORT void linear_probabilistic_count ( uint8_t *  bitmap,
const uint32_t  bitmap_bytes,
const uint8_t *  key_bytes,
const uint32_t  key_len 
)

Definition at line 1214 of file cuda_mapd_rt.cu.

References MurmurHash3().

1217  {
1218  const uint32_t bit_pos = MurmurHash3(key_bytes, key_len, 0) % (bitmap_bytes * 8);
1219  const uint32_t word_idx = bit_pos / 32;
1220  const uint32_t bit_idx = bit_pos % 32;
1221  atomicOr(((uint32_t*)bitmap) + word_idx, 1 << bit_idx);
1222 }
RUNTIME_EXPORT NEVER_INLINE DEVICE uint32_t MurmurHash3(const void *key, int len, const uint32_t seed)
Definition: MurmurHash.cpp:33

+ Here is the call graph for this function: