OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PerfectJoinHashTableBuilder Class Reference

#include <PerfectHashTableBuilder.h>

Public Member Functions

 PerfectJoinHashTableBuilder ()
 
void allocateDeviceMemory (const size_t num_column_elems, const HashType layout, HashEntryInfo hash_entry_info, const size_t shard_count, const int device_id, const int device_count, const Executor *executor)
 
void allocateDeviceMemory (const JoinColumn &join_column, const HashType layout, HashEntryInfo &hash_entry_info, const size_t shard_count, const int device_id, const int device_count, const Executor *executor)
 
void initOneToOneHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const InnerOuter &cols, const StringDictionaryProxy::IdMap *str_proxy_translation_map, const JoinType join_type, const HashType hash_type, const HashEntryInfo hash_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
void initOneToManyHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &cols, const StringDictionaryProxy::IdMap *str_proxy_translation_map, const HashEntryInfo hash_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
std::unique_ptr< PerfectHashTablegetHashTable ()
 
const bool for_semi_anti_join (const JoinType join_type)
 

Static Public Member Functions

static size_t get_entries_per_shard (const size_t total_entry_count, const size_t shard_count)
 

Private Attributes

std::unique_ptr< PerfectHashTablehash_table_
 

Detailed Description

Definition at line 24 of file PerfectHashTableBuilder.h.

Constructor & Destructor Documentation

PerfectJoinHashTableBuilder::PerfectJoinHashTableBuilder ( )
inline

Definition at line 26 of file PerfectHashTableBuilder.h.

26 {}

Member Function Documentation

void PerfectJoinHashTableBuilder::allocateDeviceMemory ( const size_t  num_column_elems,
const HashType  layout,
HashEntryInfo  hash_entry_info,
const size_t  shard_count,
const int  device_id,
const int  device_count,
const Executor executor 
)
inline

Definition at line 28 of file PerfectHashTableBuilder.h.

References CHECK, CHECK_GT, get_entries_per_shard(), Executor::getDataMgr(), HashEntryInfo::getNormalizedHashEntryCount(), GPU, HashEntryInfo::hash_entry_count, hash_table_, OneToOne, and UNREACHABLE.

Referenced by allocateDeviceMemory(), PerfectJoinHashTable::copyCpuHashTableToGpu(), and PerfectJoinHashTable::initHashTableForDevice().

34  {
35 #ifdef HAVE_CUDA
36  if (shard_count) {
37  const auto shards_per_device = (shard_count + device_count - 1) / device_count;
38  CHECK_GT(shards_per_device, 0u);
39  const size_t entries_per_shard =
40  get_entries_per_shard(hash_entry_info.hash_entry_count, shard_count);
41  hash_entry_info.hash_entry_count = entries_per_shard * shards_per_device;
42  }
43  const size_t total_count =
44  layout == HashType::OneToOne
45  ? hash_entry_info.getNormalizedHashEntryCount()
46  : 2 * hash_entry_info.getNormalizedHashEntryCount() + num_column_elems;
48  hash_table_ =
49  std::make_unique<PerfectHashTable>(executor->getDataMgr(),
50  layout,
52  hash_entry_info.getNormalizedHashEntryCount(),
53  num_column_elems);
54  hash_table_->allocateGpuMemory(total_count, device_id);
55 #else
56  UNREACHABLE();
57 #endif // HAVE_CUDA
58  }
#define UNREACHABLE()
Definition: Logger.h:266
static size_t get_entries_per_shard(const size_t total_entry_count, const size_t shard_count)
#define CHECK_GT(x, y)
Definition: Logger.h:234
size_t hash_entry_count
Data_Namespace::DataMgr * getDataMgr() const
Definition: Execute.h:577
size_t getNormalizedHashEntryCount() const
#define CHECK(condition)
Definition: Logger.h:222
std::unique_ptr< PerfectHashTable > hash_table_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::allocateDeviceMemory ( const JoinColumn join_column,
const HashType  layout,
HashEntryInfo hash_entry_info,
const size_t  shard_count,
const int  device_id,
const int  device_count,
const Executor executor 
)
inline

Definition at line 60 of file PerfectHashTableBuilder.h.

References allocateDeviceMemory(), and JoinColumn::num_elems.

66  {
67  allocateDeviceMemory(join_column.num_elems,
68  layout,
69  hash_entry_info,
70  shard_count,
71  device_id,
72  device_count,
73  executor);
74  }
size_t num_elems
void allocateDeviceMemory(const size_t num_column_elems, const HashType layout, HashEntryInfo hash_entry_info, const size_t shard_count, const int device_id, const int device_count, const Executor *executor)

+ Here is the call graph for this function:

const bool PerfectJoinHashTableBuilder::for_semi_anti_join ( const JoinType  join_type)
inline

Definition at line 394 of file PerfectHashTableBuilder.h.

References ANTI, and SEMI.

Referenced by initOneToOneHashTableOnCpu().

394  {
395  return join_type == JoinType::SEMI || join_type == JoinType::ANTI;
396  }

+ Here is the caller graph for this function:

static size_t PerfectJoinHashTableBuilder::get_entries_per_shard ( const size_t  total_entry_count,
const size_t  shard_count 
)
inlinestatic

Definition at line 388 of file PerfectHashTableBuilder.h.

References CHECK_NE.

Referenced by allocateDeviceMemory().

389  {
390  CHECK_NE(size_t(0), shard_count);
391  return (total_entry_count + shard_count - 1) / shard_count;
392  }
#define CHECK_NE(x, y)
Definition: Logger.h:231

+ Here is the caller graph for this function:

std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::getHashTable ( )
inline

Definition at line 386 of file PerfectHashTableBuilder.h.

References hash_table_.

Referenced by PerfectJoinHashTable::copyCpuHashTableToGpu(), and PerfectJoinHashTable::initHashTableForDevice().

386 { return std::move(hash_table_); }
std::unique_ptr< PerfectHashTable > hash_table_

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToManyHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &  cols,
const StringDictionaryProxy::IdMap str_proxy_translation_map,
const HashEntryInfo  hash_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 296 of file PerfectHashTableBuilder.h.

References threading_serial::async(), CHECK, CPU, cpu_threads(), StringDictionaryProxy::TranslationMap< T >::data(), DEBUG_TIMER, StringDictionaryProxy::TranslationMap< T >::domainStart(), fill_one_to_many_hash_table(), fill_one_to_many_hash_table_bucketized(), get_join_column_type_kind(), Executor::getDataMgr(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), HashEntryInfo::getNormalizedHashEntryCount(), hash_table_, init_hash_join_buff(), inline_fixed_encoding_null_val(), kDATE, JoinColumn::num_elems, and OneToMany.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

304  {
305  auto timer = DEBUG_TIMER(__func__);
306  const auto inner_col = cols.first;
307  CHECK(inner_col);
308  const auto& ti = inner_col->get_type_info();
309  CHECK(!hash_table_);
310  hash_table_ =
311  std::make_unique<PerfectHashTable>(executor->getDataMgr(),
314  hash_entry_info.getNormalizedHashEntryCount(),
315  join_column.num_elems);
316 
317  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
318 
319  int thread_count = cpu_threads();
320  {
321  auto timer_init =
322  DEBUG_TIMER("CPU One-To-Many Perfect Hash Table Builder: init_hash_join_buff");
323 #ifdef HAVE_TBB
324  init_hash_join_buff_tbb(cpu_hash_table_buff,
325  hash_entry_info.getNormalizedHashEntryCount(),
326  hash_join_invalid_val);
327 #else // #ifdef HAVE_TBB
328  std::vector<std::future<void> > init_threads;
329  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
330  init_threads.emplace_back(
333  cpu_hash_table_buff,
334  hash_entry_info.getNormalizedHashEntryCount(),
335  hash_join_invalid_val,
336  thread_idx,
337  thread_count));
338  }
339  for (auto& child : init_threads) {
340  child.wait();
341  }
342  for (auto& child : init_threads) {
343  child.get();
344  }
345 #endif // !HAVE_TBB
346  }
347  {
348  auto timer_fill = DEBUG_TIMER(
349  "CPU One-To-Many Perfect Hash Table Builder: fill_hash_join_buff_bucketized");
350  if (ti.get_type() == kDATE) {
352  cpu_hash_table_buff,
353  hash_entry_info,
354  join_column,
355  {static_cast<size_t>(ti.get_size()),
356  col_range.getIntMin(),
357  col_range.getIntMax(),
359  is_bitwise_eq,
360  col_range.getIntMax() + 1,
362  str_proxy_translation_map ? str_proxy_translation_map->data() : nullptr,
363  str_proxy_translation_map ? str_proxy_translation_map->domainStart()
364  : 0 /*dummy*/,
365  thread_count);
366  } else {
368  cpu_hash_table_buff,
369  hash_entry_info,
370  join_column,
371  {static_cast<size_t>(ti.get_size()),
372  col_range.getIntMin(),
373  col_range.getIntMax(),
375  is_bitwise_eq,
376  col_range.getIntMax() + 1,
378  str_proxy_translation_map ? str_proxy_translation_map->data() : nullptr,
379  str_proxy_translation_map ? str_proxy_translation_map->domainStart()
380  : 0 /*dummy*/,
381  thread_count);
382  }
383  }
384  }
int64_t getIntMin() const
size_t num_elems
future< Result > async(Fn &&fn, Args &&...args)
void fill_one_to_many_hash_table_bucketized(int32_t *buff, const HashEntryInfo hash_entry_info, const JoinColumn &join_column, const JoinColumnTypeInfo &type_info, const int32_t *sd_inner_to_outer_translation_map, const int32_t min_inner_elem, const unsigned cpu_thread_count)
Definition: sqltypes.h:68
void fill_one_to_many_hash_table(int32_t *buff, const HashEntryInfo hash_entry_info, const JoinColumn &join_column, const JoinColumnTypeInfo &type_info, const int32_t *sd_inner_to_outer_translation_map, const int32_t min_inner_elem, const unsigned cpu_thread_count)
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
Data_Namespace::DataMgr * getDataMgr() const
Definition: Execute.h:577
size_t getNormalizedHashEntryCount() const
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:222
#define DEBUG_TIMER(name)
Definition: Logger.h:371
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::unique_ptr< PerfectHashTable > hash_table_
int cpu_threads()
Definition: thread_count.h:25

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToOneHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const InnerOuter cols,
const StringDictionaryProxy::IdMap str_proxy_translation_map,
const JoinType  join_type,
const HashType  hash_type,
const HashEntryInfo  hash_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 191 of file PerfectHashTableBuilder.h.

References HashEntryInfo::bucket_normalization, CHECK, CPU, cpu_threads(), StringDictionaryProxy::TranslationMap< T >::data(), DEBUG_TIMER, StringDictionaryProxy::TranslationMap< T >::domainStart(), fill_hash_join_buff_bucketized(), for_semi_anti_join(), get_join_column_type_kind(), Executor::getDataMgr(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), HashEntryInfo::getNormalizedHashEntryCount(), hash_table_, init_hash_join_buff(), and inline_fixed_encoding_null_val().

Referenced by PerfectJoinHashTable::initHashTableForDevice().

201  {
202  auto timer = DEBUG_TIMER(__func__);
203  const auto inner_col = cols.first;
204  CHECK(inner_col);
205  const auto& ti = inner_col->get_type_info();
206 
207  CHECK(!hash_table_);
208  hash_table_ =
209  std::make_unique<PerfectHashTable>(executor->getDataMgr(),
210  hash_type,
212  hash_entry_info.getNormalizedHashEntryCount(),
213  0);
214 
215  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
216  const int thread_count = cpu_threads();
217  std::vector<std::thread> init_cpu_buff_threads;
218 
219  {
220  auto timer_init = DEBUG_TIMER("CPU One-To-One Perfect-Hash: init_hash_join_buff");
221 #ifdef HAVE_TBB
222  init_hash_join_buff_tbb(cpu_hash_table_buff,
223  hash_entry_info.getNormalizedHashEntryCount(),
224  hash_join_invalid_val);
225 #else // #ifdef HAVE_TBB
226  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
227  init_cpu_buff_threads.emplace_back([hash_entry_info,
228  hash_join_invalid_val,
229  thread_idx,
230  thread_count,
231  cpu_hash_table_buff] {
232  init_hash_join_buff(cpu_hash_table_buff,
233  hash_entry_info.getNormalizedHashEntryCount(),
234  hash_join_invalid_val,
235  thread_idx,
236  thread_count);
237  });
238  }
239  for (auto& t : init_cpu_buff_threads) {
240  t.join();
241  }
242  init_cpu_buff_threads.clear();
243 #endif // !HAVE_TBB
244  }
245  const bool for_semi_join = for_semi_anti_join(join_type);
246  std::atomic<int> err{0};
247  {
248  auto timer_fill =
249  DEBUG_TIMER("CPU One-To-One Perfect-Hash: fill_hash_join_buff_bucketized");
250  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
251  init_cpu_buff_threads.emplace_back([hash_join_invalid_val,
252  &join_column,
253  str_proxy_translation_map,
254  thread_idx,
255  thread_count,
256  &ti,
257  &err,
258  &col_range,
259  &is_bitwise_eq,
260  &for_semi_join,
261  cpu_hash_table_buff,
262  hash_entry_info] {
263  int partial_err = fill_hash_join_buff_bucketized(
264  cpu_hash_table_buff,
265  hash_join_invalid_val,
266  for_semi_join,
267  join_column,
268  {static_cast<size_t>(ti.get_size()),
269  col_range.getIntMin(),
270  col_range.getIntMax(),
272  is_bitwise_eq,
273  col_range.getIntMax() + 1,
275  str_proxy_translation_map ? str_proxy_translation_map->data() : nullptr,
276  str_proxy_translation_map ? str_proxy_translation_map->domainStart()
277  : 0, // 0 is dummy value
278  thread_idx,
279  thread_count,
280  hash_entry_info.bucket_normalization);
281  int zero{0};
282  err.compare_exchange_strong(zero, partial_err);
283  });
284  }
285  for (auto& t : init_cpu_buff_threads) {
286  t.join();
287  }
288  }
289  if (err) {
290  // Too many hash entries, need to retry with a 1:many table
291  hash_table_ = nullptr; // clear the hash table buffer
292  throw NeedsOneToManyHash();
293  }
294  }
int64_t getIntMin() const
DEVICE int SUFFIX() fill_hash_join_buff_bucketized(int32_t *buff, const int32_t invalid_slot_val, const bool for_semi_join, const JoinColumn join_column, const JoinColumnTypeInfo type_info, const int32_t *sd_inner_to_outer_translation_map, const int32_t min_inner_elem, const int32_t cpu_thread_idx, const int32_t cpu_thread_count, const int64_t bucket_normalization)
const bool for_semi_anti_join(const JoinType join_type)
int64_t bucket_normalization
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
Data_Namespace::DataMgr * getDataMgr() const
Definition: Execute.h:577
size_t getNormalizedHashEntryCount() const
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:222
#define DEBUG_TIMER(name)
Definition: Logger.h:371
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::unique_ptr< PerfectHashTable > hash_table_
int cpu_threads()
Definition: thread_count.h:25

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::hash_table_
private

The documentation for this class was generated from the following file: