RuntimeFunctions.h File Reference

#include <cassert>
#include <cstdint>
#include <ctime>
#include <limits>
#include <type_traits>

Include dependency graph for RuntimeFunctions.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	EMPTY_KEY_64   std::numeric_limits<int64_t>::max()
#define	EMPTY_KEY_32   std::numeric_limits<int32_t>::max()
#define	EMPTY_KEY_16   std::numeric_limits<int16_t>::max()
#define	EMPTY_KEY_8   std::numeric_limits<int8_t>::max()

Functions
int64_t	agg_sum (int64_t *agg, const int64_t val)
void	agg_max (int64_t *agg, const int64_t val)
void	agg_min (int64_t *agg, const int64_t val)
void	agg_sum_double (int64_t *agg, const double val)
void	agg_max_double (int64_t *agg, const double val)
void	agg_min_double (int64_t *agg, const double val)
int32_t	agg_sum_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val)
int64_t	agg_sum_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val)
void	agg_max_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val)
void	agg_min_skip_val (int64_t *agg, const int64_t val, const int64_t skip_val)
void	agg_sum_float_skip_val (int32_t *agg, const float val, const float skip_val)
void	agg_sum_double_skip_val (int64_t *agg, const double val, const double skip_val)
void	agg_max_double_skip_val (int64_t *agg, const double val, const double skip_val)
void	agg_min_double_skip_val (int64_t *agg, const double val, const double skip_val)
int32_t	agg_sum_int32 (int32_t *agg, const int32_t val)
void	agg_max_int32 (int32_t *agg, const int32_t val)
void	agg_max_int16 (int16_t *agg, const int16_t val)
void	agg_max_int8 (int8_t *agg, const int8_t val)
void	agg_min_int32 (int32_t *agg, const int32_t val)
void	agg_min_int16 (int16_t *agg, const int16_t val)
void	agg_min_int8 (int8_t *agg, const int8_t val)
void	agg_sum_float (int32_t *agg, const float val)
void	agg_max_float (int32_t *agg, const float val)
void	agg_min_float (int32_t *agg, const float val)
void	agg_max_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val)
void	agg_max_int16_skip_val (int16_t *agg, const int16_t val, const int16_t skip_val)
void	agg_max_int8_skip_val (int8_t *agg, const int8_t val, const int8_t skip_val)
void	agg_min_int32_skip_val (int32_t *agg, const int32_t val, const int32_t skip_val)
void	agg_min_int16_skip_val (int16_t *agg, const int16_t val, const int16_t skip_val)
void	agg_min_int8_skip_val (int8_t *agg, const int8_t val, const int8_t skip_val)
void	agg_max_float_skip_val (int32_t *agg, const float val, const float skip_val)
void	agg_min_float_skip_val (int32_t *agg, const float val, const float skip_val)
void	agg_count_distinct_bitmap (int64_t *agg, const int64_t val, const int64_t min_val)
uint32_t	key_hash (const int64_t *key, const uint32_t key_qw_count, const uint32_t key_byte_width)
int64_t *	get_group_value (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad, const int64_t *init_val=nullptr)
int64_t *	get_group_value_with_watchdog (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const uint32_t key_width, const uint32_t row_size_quad, const int64_t *init_val=nullptr)
int64_t *	get_group_value_columnar (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_qw_count)
int64_t *	get_group_value_columnar_with_watchdog (int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_qw_count)
int64_t *	get_group_value_fast (int64_t *groups_buffer, const int64_t key, const int64_t min_key, const int64_t bucket, const uint32_t row_size_quad)
int64_t *	get_group_value_fast_with_original_key (int64_t *groups_buffer, const int64_t key, const int64_t orig_key, const int64_t min_key, const int64_t bucket, const uint32_t row_size_quad)
uint32_t	get_columnar_group_bin_offset (int64_t *key_base_ptr, const int64_t key, const int64_t min_key, const int64_t bucket)
int64_t *	get_matching_group_value_perfect_hash (int64_t *groups_buffer, const uint32_t h, const int64_t *key, const uint32_t key_qw_count, const uint32_t row_size_quad)
int32_t *	get_bucketized_hash_slot (int32_t *buff, const int64_t key, const int64_t min_key, const int64_t bucket_normalization=1)
int32_t *	get_hash_slot (int32_t *buff, const int64_t key, const int64_t min_key)
int32_t *	get_hash_slot_sharded (int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t num_shards, const uint32_t device_count)
int32_t *	get_bucketized_hash_slot_sharded (int32_t *buff, const int64_t key, const int64_t min_key, const uint32_t entry_count_per_shard, const uint32_t num_shards, const uint32_t device_count, const int64_t bucket_normalization)
void	linear_probabilistic_count (uint8_t *bitmap, const uint32_t bitmap_bytes, const uint8_t *key_bytes, const uint32_t key_len)
int64_t	fixed_width_int_decode_noinline (const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
int64_t	fixed_width_unsigned_decode_noinline (const int8_t *byte_stream, const int32_t byte_width, const int64_t pos)
float	fixed_width_float_decode_noinline (const int8_t *byte_stream, const int64_t pos)
double	fixed_width_double_decode_noinline (const int8_t *byte_stream, const int64_t pos)
int64_t	fixed_width_small_date_decode_noinline (const int8_t *byte_stream, const int32_t byte_width, const int32_t null_val, const int64_t ret_null_val, const int64_t pos)
int8_t *	extract_str_ptr_noinline (const uint64_t str_and_len)
int32_t	extract_str_len_noinline (const uint64_t str_and_len)
template<typename T = int64_t>
T	get_empty_key ()
template<>
int32_t	get_empty_key ()

Macro Definition Documentation

EMPTY_KEY_16

#define EMPTY_KEY_16   std::numeric_limits<int16_t>::max()

Definition at line 116 of file RuntimeFunctions.h.

EMPTY_KEY_32

#define EMPTY_KEY_32   std::numeric_limits<int32_t>::max()

Definition at line 115 of file RuntimeFunctions.h.

Referenced by get_empty_key().

EMPTY_KEY_64

#define EMPTY_KEY_64   std::numeric_limits<int64_t>::max()

Definition at line 114 of file RuntimeFunctions.h.

Referenced by get_empty_key().

EMPTY_KEY_8

#define EMPTY_KEY_8   std::numeric_limits<int8_t>::max()

Definition at line 117 of file RuntimeFunctions.h.

Function Documentation

agg_count_distinct_bitmap()

void agg_count_distinct_bitmap	(	int64_t *	agg,
		const int64_t	val,
		const int64_t	min_val
	)

Definition at line 284 of file RuntimeFunctions.cpp.

Referenced by agg_count_distinct_bitmap_skip_val(), WindowFunctionContext::fillPartitionEnd(), WindowFunctionContext::fillPartitionStart(), anonymous_namespace{WindowContext.cpp}::index_to_partition_end(), and InValuesBitmap::InValuesBitmap().

                                                                               {
  const uint64_t bitmap_idx = val - min_val;
  reinterpret_cast<int8_t*>(*agg)[bitmap_idx >> 3] |= (1 << (bitmap_idx & 7));
}

Here is the caller graph for this function:

agg_max()

void agg_max	(	int64_t *	agg,
		const int64_t	val
	)

Definition at line 345 of file RuntimeFunctions.cpp.

Referenced by agg_count_int32_skip_val().

                                                                       {
  *agg = std::max(*agg, val);
}

Here is the caller graph for this function:

agg_max_double()

void agg_max_double	(	int64_t *	agg,
		const double	val
	)

Definition at line 569 of file RuntimeFunctions.cpp.

                                                                             {
  const auto r = std::max(*reinterpret_cast<const double*>(agg), val);
  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&r)));
}

agg_max_double_skip_val()

void agg_max_double_skip_val	(	int64_t *	agg,
		const double	val,
		const double	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_max_float()

void agg_max_float	(	int32_t *	agg,
		const float	val
	)

Definition at line 610 of file RuntimeFunctions.cpp.

                                                                           {
  const auto r = std::max(*reinterpret_cast<const float*>(agg), val);
  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&r)));
}

agg_max_float_skip_val()

void agg_max_float_skip_val	(	int32_t *	agg,
		const float	val,
		const float	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_max_int16()

void agg_max_int16	(	int16_t *	agg,
		const int16_t	val
	)

Referenced by agg_count_int32_skip_val().

Here is the caller graph for this function:

agg_max_int16_skip_val()

void agg_max_int16_skip_val	(	int16_t *	agg,
		const int16_t	val,
		const int16_t	skip_val
	)

agg_max_int32()

void agg_max_int32	(	int32_t *	agg,
		const int32_t	val
	)

Referenced by agg_count_int32_skip_val().

Here is the caller graph for this function:

agg_max_int32_skip_val()

void agg_max_int32_skip_val	(	int32_t *	agg,
		const int32_t	val,
		const int32_t	skip_val
	)

agg_max_int8()

void agg_max_int8	(	int8_t *	agg,
		const int8_t	val
	)

Referenced by agg_count_int32_skip_val().

Here is the caller graph for this function:

agg_max_int8_skip_val()

void agg_max_int8_skip_val	(	int8_t *	agg,
		const int8_t	val,
		const int8_t	skip_val
	)

agg_max_skip_val()

void agg_max_skip_val	(	int64_t *	agg,
		const int64_t	val,
		const int64_t	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_min()

void agg_min	(	int64_t *	agg,
		const int64_t	val
	)

Definition at line 349 of file RuntimeFunctions.cpp.

Referenced by agg_count_int32_skip_val().

                                                                       {
  *agg = std::min(*agg, val);
}

Here is the caller graph for this function:

agg_min_double()

void agg_min_double	(	int64_t *	agg,
		const double	val
	)

Definition at line 574 of file RuntimeFunctions.cpp.

                                                                             {
  const auto r = std::min(*reinterpret_cast<const double*>(agg), val);
  *agg = *(reinterpret_cast<const int64_t*>(may_alias_ptr(&r)));
}

agg_min_double_skip_val()

void agg_min_double_skip_val	(	int64_t *	agg,
		const double	val,
		const double	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_min_float()

void agg_min_float	(	int32_t *	agg,
		const float	val
	)

Definition at line 615 of file RuntimeFunctions.cpp.

                                                                           {
  const auto r = std::min(*reinterpret_cast<const float*>(agg), val);
  *agg = *(reinterpret_cast<const int32_t*>(may_alias_ptr(&r)));
}

agg_min_float_skip_val()

void agg_min_float_skip_val	(	int32_t *	agg,
		const float	val,
		const float	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_min_int16()

void agg_min_int16	(	int16_t *	agg,
		const int16_t	val
	)

Referenced by agg_count_int32_skip_val().

Here is the caller graph for this function:

agg_min_int16_skip_val()

void agg_min_int16_skip_val	(	int16_t *	agg,
		const int16_t	val,
		const int16_t	skip_val
	)

agg_min_int32()

void agg_min_int32	(	int32_t *	agg,
		const int32_t	val
	)

Referenced by agg_count_int32_skip_val().

Here is the caller graph for this function:

agg_min_int32_skip_val()

void agg_min_int32_skip_val	(	int32_t *	agg,
		const int32_t	val,
		const int32_t	skip_val
	)

agg_min_int8()

void agg_min_int8	(	int8_t *	agg,
		const int8_t	val
	)

Referenced by agg_count_int32_skip_val().

Here is the caller graph for this function:

agg_min_int8_skip_val()

void agg_min_int8_skip_val	(	int8_t *	agg,
		const int8_t	val,
		const int8_t	skip_val
	)

agg_min_skip_val()

void agg_min_skip_val	(	int64_t *	agg,
		const int64_t	val,
		const int64_t	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_sum()

int64_t agg_sum	(	int64_t *	agg,
		const int64_t	val
	)

Definition at line 339 of file RuntimeFunctions.cpp.

Referenced by agg_sum_skip_val().

                                                                          {
  const auto old = *agg;
  *agg += val;
  return old;
}

Here is the caller graph for this function:

agg_sum_double()

void agg_sum_double	(	int64_t *	agg,
		const double	val
	)

Definition at line 564 of file RuntimeFunctions.cpp.

                                                                             {
  const auto r = *reinterpret_cast<const double*>(agg) + val;
  *agg = *reinterpret_cast<const int64_t*>(may_alias_ptr(&r));
}

agg_sum_double_skip_val()

void agg_sum_double_skip_val	(	int64_t *	agg,
		const double	val,
		const double	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_sum_float()

void agg_sum_float	(	int32_t *	agg,
		const float	val
	)

Definition at line 605 of file RuntimeFunctions.cpp.

                                                                           {
  const auto r = *reinterpret_cast<const float*>(agg) + val;
  *agg = *reinterpret_cast<const int32_t*>(may_alias_ptr(&r));
}

agg_sum_float_skip_val()

void agg_sum_float_skip_val	(	int32_t *	agg,
		const float	val,
		const float	skip_val
	)

Referenced by Executor::reduceResults().

Here is the caller graph for this function:

agg_sum_int32()

int32_t agg_sum_int32	(	int32_t *	agg,
		const int32_t	val
	)

Definition at line 397 of file RuntimeFunctions.cpp.

Referenced by agg_sum_int32_skip_val().

                                                                                {
  const auto old = *agg;
  *agg += val;
  return old;
}

Here is the caller graph for this function:

agg_sum_int32_skip_val()

int32_t agg_sum_int32_skip_val	(	int32_t *	agg,
		const int32_t	val,
		const int32_t	skip_val
	)

Definition at line 482 of file RuntimeFunctions.cpp.

References agg_sum_int32().

                                                                                {
  const auto old = *agg;
  if (val != skip_val) {
    if (old != skip_val) {
      return agg_sum_int32(agg, val);
    } else {
      *agg = val;
    }
  }
  return old;
}

Here is the call graph for this function:

agg_sum_skip_val()

int64_t agg_sum_skip_val	(	int64_t *	agg,
		const int64_t	val,
		const int64_t	skip_val
	)

Definition at line 468 of file RuntimeFunctions.cpp.

References agg_sum().

Referenced by Executor::reduceResults().

                                                                          {
  const auto old = *agg;
  if (val != skip_val) {
    if (old != skip_val) {
      return agg_sum(agg, val);
    } else {
      *agg = val;
    }
  }
  return old;
}

Here is the call graph for this function:

Here is the caller graph for this function:

extract_str_len_noinline()

int32_t extract_str_len_noinline

(

const uint64_t

str_and_len

)

Referenced by extract_str_len(), and string_compress().

Here is the caller graph for this function:

extract_str_ptr_noinline()

int8_t* extract_str_ptr_noinline

(

const uint64_t

str_and_len

)

Referenced by extract_str_len(), and string_compress().

Here is the caller graph for this function:

fixed_width_double_decode_noinline()

double fixed_width_double_decode_noinline	(	const int8_t *	byte_stream,
		const int64_t	pos
	)

Definition at line 126 of file DecodersImpl.h.

References ALWAYS_INLINE, DEVICE, fixed_width_double_decode(), and SUFFIX.

Referenced by compute_bucket_sizes_impl(), lazy_decode(), and OverlapsKeyHandler::operator()().

                                                                                      {
  return SUFFIX(fixed_width_double_decode)(byte_stream, pos);
}

Here is the call graph for this function:

Here is the caller graph for this function:

fixed_width_float_decode_noinline()

float fixed_width_float_decode_noinline	(	const int8_t *	byte_stream,
		const int64_t	pos
	)

Definition at line 113 of file DecodersImpl.h.

References ALWAYS_INLINE, DEVICE, fixed_width_float_decode(), and SUFFIX.

Referenced by lazy_decode().

                                                                                     {
  return SUFFIX(fixed_width_float_decode)(byte_stream, pos);
}

Here is the call graph for this function:

Here is the caller graph for this function:

fixed_width_int_decode_noinline()

int64_t fixed_width_int_decode_noinline	(	const int8_t *	byte_stream,
		const int32_t	byte_width,
		const int64_t	pos
	)

Definition at line 83 of file DecodersImpl.h.

References DEVICE, fixed_width_int_decode(), NEVER_INLINE, and SUFFIX.

Referenced by get_join_column_element_value(), and lazy_decode().

                                                           {
  return SUFFIX(fixed_width_int_decode)(byte_stream, byte_width, pos);
}

Here is the call graph for this function:

Here is the caller graph for this function:

fixed_width_small_date_decode_noinline()

int64_t fixed_width_small_date_decode_noinline	(	const int8_t *	byte_stream,
		const int32_t	byte_width,
		const int32_t	null_val,
		const int64_t	ret_null_val,
		const int64_t	pos
	)

Definition at line 141 of file DecodersImpl.h.

References fixed_width_small_date_decode(), and SUFFIX.

Referenced by get_join_column_element_value(), and lazy_decode().

                                                                  {
  return SUFFIX(fixed_width_small_date_decode)(
      byte_stream, byte_width, null_val, ret_null_val, pos);
}

Here is the call graph for this function:

Here is the caller graph for this function:

fixed_width_unsigned_decode_noinline()

int64_t fixed_width_unsigned_decode_noinline	(	const int8_t *	byte_stream,
		const int32_t	byte_width,
		const int64_t	pos
	)

Definition at line 90 of file DecodersImpl.h.

References ALWAYS_INLINE, DEVICE, fixed_width_unsigned_decode(), and SUFFIX.

Referenced by get_join_column_element_value(), and lazy_decode().

                                                                {
  return SUFFIX(fixed_width_unsigned_decode)(byte_stream, byte_width, pos);
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_bucketized_hash_slot()

int32_t* get_bucketized_hash_slot	(	int32_t *	buff,
		const int64_t	key,
		const int64_t	min_key,
		const int64_t	bucket_normalization = 1
	)

Definition at line 31 of file JoinHashImpl.h.

Referenced by bucketized_hash_join_idx(), count_matches_bucketized(), fill_hash_join_buff_bucketized(), and fill_row_ids_bucketized().

                                        {
  return buff + (key - min_key) / bucket_normalization;
}

Here is the caller graph for this function:

get_bucketized_hash_slot_sharded()

int32_t* get_bucketized_hash_slot_sharded	(	int32_t *	buff,
		const int64_t	key,
		const int64_t	min_key,
		const uint32_t	entry_count_per_shard,
		const uint32_t	num_shards,
		const uint32_t	device_count,
		const int64_t	bucket_normalization
	)

Definition at line 45 of file JoinHashImpl.h.

References SHARD_FOR_KEY.

Referenced by fill_hash_join_buff_sharded_bucketized(), and fill_row_ids_sharded_bucketized().

                                        {
  const uint32_t shard = SHARD_FOR_KEY(key, num_shards);
  const uint32_t shard_buffer_index =
      shard / device_count;  // shard sub-buffer index within `buff`
  int32_t* shard_buffer = buff + shard_buffer_index * entry_count_per_shard;
  return shard_buffer + (key - min_key) / bucket_normalization / num_shards;
}

Here is the caller graph for this function:

get_columnar_group_bin_offset()

uint32_t get_columnar_group_bin_offset	(	int64_t *	key_base_ptr,
		const int64_t	key,
		const int64_t	min_key,
		const int64_t	bucket
	)

Definition at line 231 of file GroupByRuntime.cpp.

References EMPTY_KEY_64.

                                                    {
  int64_t off = key - min_key;
  if (bucket) {
    off /= bucket;
  }
  if (key_base_ptr[off] == EMPTY_KEY_64) {
    key_base_ptr[off] = key;
  }
  return off;
}

get_empty_key() [1/2]

template<typename T = int64_t>

T get_empty_key ( )

inline

Definition at line 232 of file RuntimeFunctions.h.

References EMPTY_KEY_64.

                         {
  static_assert(std::is_same<T, int64_t>::value,
                "Unsupported template parameter other than int64_t for now");
  return EMPTY_KEY_64;
}

get_empty_key() [2/2]

template<>

int32_t get_empty_key ( )

inline

Definition at line 239 of file RuntimeFunctions.h.

References EMPTY_KEY_32.

                               {
  return EMPTY_KEY_32;
}

get_group_value()

int64_t* get_group_value	(	int64_t *	groups_buffer,
		const uint32_t	groups_buffer_entry_count,
		const int64_t *	key,
		const uint32_t	key_count,
		const uint32_t	key_width,
		const uint32_t	row_size_quad,
		const int64_t *	init_val = nullptr
	)

Definition at line 26 of file GroupByRuntime.cpp.

References DEVICE, dynamic_watchdog(), get_matching_group_value(), groups_buffer_entry_count, key_hash(), and NEVER_INLINE.

Referenced by ResultSetStorage::moveOneEntryToBuffer().

                              {
  uint32_t h = key_hash(key, key_count, key_width) % groups_buffer_entry_count;
  int64_t* matching_group = get_matching_group_value(
      groups_buffer, h, key, key_count, key_width, row_size_quad, init_vals);
  if (matching_group) {
    return matching_group;
  }
  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
  while (h_probe != h) {
    matching_group = get_matching_group_value(
        groups_buffer, h_probe, key, key_count, key_width, row_size_quad, init_vals);
    if (matching_group) {
      return matching_group;
    }
    h_probe = (h_probe + 1) % groups_buffer_entry_count;
  }
  return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_group_value_columnar()

int64_t* get_group_value_columnar	(	int64_t *	groups_buffer,
		const uint32_t	groups_buffer_entry_count,
		const int64_t *	key,
		const uint32_t	key_qw_count
	)

Definition at line 142 of file GroupByRuntime.cpp.

References get_matching_group_value_columnar(), groups_buffer_entry_count, and key_hash().

Referenced by ResultSetStorage::moveOneEntryToBuffer().

                                 {
  uint32_t h = key_hash(key, key_qw_count, sizeof(int64_t)) % groups_buffer_entry_count;
  int64_t* matching_group = get_matching_group_value_columnar(
      groups_buffer, h, key, key_qw_count, groups_buffer_entry_count);
  if (matching_group) {
    return matching_group;
  }
  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
  while (h_probe != h) {
    matching_group = get_matching_group_value_columnar(
        groups_buffer, h_probe, key, key_qw_count, groups_buffer_entry_count);
    if (matching_group) {
      return matching_group;
    }
    h_probe = (h_probe + 1) % groups_buffer_entry_count;
  }
  return NULL;
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_group_value_columnar_with_watchdog()

int64_t* get_group_value_columnar_with_watchdog	(	int64_t *	groups_buffer,
		const uint32_t	groups_buffer_entry_count,
		const int64_t *	key,
		const uint32_t	key_qw_count
	)

Definition at line 165 of file GroupByRuntime.cpp.

References dynamic_watchdog(), get_matching_group_value_columnar(), groups_buffer_entry_count, and key_hash().

                                 {
  uint32_t h = key_hash(key, key_qw_count, sizeof(int64_t)) % groups_buffer_entry_count;
  int64_t* matching_group = get_matching_group_value_columnar(
      groups_buffer, h, key, key_qw_count, groups_buffer_entry_count);
  if (matching_group) {
    return matching_group;
  }
  uint32_t watchdog_countdown = 100;
  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
  while (h_probe != h) {
    matching_group = get_matching_group_value_columnar(
        groups_buffer, h_probe, key, key_qw_count, groups_buffer_entry_count);
    if (matching_group) {
      return matching_group;
    }
    h_probe = (h_probe + 1) % groups_buffer_entry_count;
    if (--watchdog_countdown == 0) {
      if (dynamic_watchdog()) {
        return NULL;
      }
      watchdog_countdown = 100;
    }
  }
  return NULL;
}

Here is the call graph for this function:

get_group_value_fast()

int64_t* get_group_value_fast	(	int64_t *	groups_buffer,
		const int64_t	key,
		const int64_t	min_key,
		const int64_t	bucket,
		const uint32_t	row_size_quad
	)

Definition at line 195 of file GroupByRuntime.cpp.

References EMPTY_KEY_64.

                                  {
  int64_t key_diff = key - min_key;
  if (bucket) {
    key_diff /= bucket;
  }
  int64_t off = key_diff * row_size_quad;
  if (groups_buffer[off] == EMPTY_KEY_64) {
    groups_buffer[off] = key;
  }
  return groups_buffer + off + 1;
}

get_group_value_fast_with_original_key()

int64_t* get_group_value_fast_with_original_key	(	int64_t *	groups_buffer,
		const int64_t	key,
		const int64_t	orig_key,
		const int64_t	min_key,
		const int64_t	bucket,
		const uint32_t	row_size_quad
	)

Definition at line 212 of file GroupByRuntime.cpp.

References ALWAYS_INLINE, DEVICE, and EMPTY_KEY_64.

                                  {
  int64_t key_diff = key - min_key;
  if (bucket) {
    key_diff /= bucket;
  }
  int64_t off = key_diff * row_size_quad;
  if (groups_buffer[off] == EMPTY_KEY_64) {
    groups_buffer[off] = orig_key;
  }
  return groups_buffer + off + 1;
}

get_group_value_with_watchdog()

int64_t* get_group_value_with_watchdog	(	int64_t *	groups_buffer,
		const uint32_t	groups_buffer_entry_count,
		const int64_t *	key,
		const uint32_t	key_count,
		const uint32_t	key_width,
		const uint32_t	row_size_quad,
		const int64_t *	init_val = nullptr
	)

Definition at line 54 of file GroupByRuntime.cpp.

References DEVICE, dynamic_watchdog(), get_matching_group_value(), groups_buffer_entry_count, key_hash(), and NEVER_INLINE.

                              {
  uint32_t h = key_hash(key, key_count, key_width) % groups_buffer_entry_count;
  int64_t* matching_group = get_matching_group_value(
      groups_buffer, h, key, key_count, key_width, row_size_quad, init_vals);
  if (matching_group) {
    return matching_group;
  }
  uint32_t watchdog_countdown = 100;
  uint32_t h_probe = (h + 1) % groups_buffer_entry_count;
  while (h_probe != h) {
    matching_group = get_matching_group_value(
        groups_buffer, h_probe, key, key_count, key_width, row_size_quad, init_vals);
    if (matching_group) {
      return matching_group;
    }
    h_probe = (h_probe + 1) % groups_buffer_entry_count;
    if (--watchdog_countdown == 0) {
      if (dynamic_watchdog()) {
        return NULL;
      }
      watchdog_countdown = 100;
    }
  }
  return NULL;
}

Here is the call graph for this function:

get_hash_slot()

int32_t* get_hash_slot	(	int32_t *	buff,
		const int64_t	key,
		const int64_t	min_key
	)

Definition at line 39 of file JoinHashImpl.h.

Referenced by count_matches(), fill_hash_join_buff(), fill_row_ids(), and hash_join_idx().

                                                                                      {
  return buff + (key - min_key);
}

Here is the caller graph for this function:

get_hash_slot_sharded()

int32_t* get_hash_slot_sharded	(	int32_t *	buff,
		const int64_t	key,
		const int64_t	min_key,
		const uint32_t	entry_count_per_shard,
		const uint32_t	num_shards,
		const uint32_t	device_count
	)

Definition at line 60 of file JoinHashImpl.h.

References SHARD_FOR_KEY.

Referenced by count_matches_sharded(), fill_hash_join_buff_sharded(), fill_row_ids_sharded(), and hash_join_idx_sharded().

                                 {
  const uint32_t shard = SHARD_FOR_KEY(key, num_shards);
  const uint32_t shard_buffer_index =
      shard / device_count;  // shard sub-buffer index within `buff`
  int32_t* shard_buffer = buff + shard_buffer_index * entry_count_per_shard;
  return shard_buffer + (key - min_key) / num_shards;
}

Here is the caller graph for this function:

get_matching_group_value_perfect_hash()

int64_t* get_matching_group_value_perfect_hash	(	int64_t *	groups_buffer,
		const uint32_t	h,
		const int64_t *	key,
		const uint32_t	key_qw_count,
		const uint32_t	row_size_quad
	)

Definition at line 1131 of file RuntimeFunctions.cpp.

References EMPTY_KEY_64.

                                  {
  uint32_t off = hashed_index * row_size_quad;
  if (groups_buffer[off] == EMPTY_KEY_64) {
    for (uint32_t i = 0; i < key_count; ++i) {
      groups_buffer[off + i] = key[i];
    }
  }
  return groups_buffer + off + key_count;
}

key_hash()

uint32_t key_hash	(	const int64_t *	key,
		const uint32_t	key_qw_count,
		const uint32_t	key_byte_width
	)

Definition at line 20 of file GroupByRuntime.cpp.

References MurmurHash1().

Referenced by get_group_value(), get_group_value_columnar(), anonymous_namespace{ResultSetReduction.cpp}::get_group_value_columnar_reduction(), get_group_value_columnar_slot(), get_group_value_columnar_slot_with_watchdog(), get_group_value_columnar_with_watchdog(), get_group_value_reduction(), and get_group_value_with_watchdog().

                                                                                 {
  return MurmurHash1(key, key_byte_width * key_count, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

linear_probabilistic_count()

void linear_probabilistic_count	(	uint8_t *	bitmap,
		const uint32_t	bitmap_bytes,
		const uint8_t *	key_bytes,
		const uint32_t	key_len
	)

Definition at line 1280 of file RuntimeFunctions.cpp.

References __attribute__(), and MurmurHash1().

                                                                                {
  const uint32_t bit_pos = MurmurHash1(key_bytes, key_len, 0) % (bitmap_bytes * 8);
  const uint32_t word_idx = bit_pos / 32;
  const uint32_t bit_idx = bit_pos % 32;
  reinterpret_cast<uint32_t*>(bitmap)[word_idx] |= 1 << bit_idx;
}

Here is the call graph for this function: