OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PerfectJoinHashTableBuilder Class Reference

#include <PerfectHashTableBuilder.h>

Public Member Functions

 PerfectJoinHashTableBuilder ()
 
void allocateDeviceMemory (BucketizedHashEntryInfo hash_entry_info, PerfectHashTableEntryInfo hash_table_entry_info, const size_t shard_count, const int device_id, const int device_count, const Executor *executor)
 
void initOneToOneHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const InnerOuter &cols, const StringDictionaryProxy::IdMap *str_proxy_translation_map, const JoinType join_type, const BucketizedHashEntryInfo hash_entry_info, const PerfectHashTableEntryInfo hash_table_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
void initOneToManyHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &cols, const StringDictionaryProxy::IdMap *str_proxy_translation_map, const JoinType join_type, const BucketizedHashEntryInfo hash_entry_info, const PerfectHashTableEntryInfo hash_table_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
std::unique_ptr< PerfectHashTablegetHashTable ()
 
const bool for_semi_anti_join (const JoinType join_type)
 

Static Public Member Functions

static size_t get_entries_per_shard (const size_t total_entry_count, const size_t shard_count)
 

Private Attributes

std::unique_ptr< PerfectHashTablehash_table_
 

Detailed Description

Definition at line 25 of file PerfectHashTableBuilder.h.

Constructor & Destructor Documentation

PerfectJoinHashTableBuilder::PerfectJoinHashTableBuilder ( )
inline

Definition at line 27 of file PerfectHashTableBuilder.h.

27 {}

Member Function Documentation

void PerfectJoinHashTableBuilder::allocateDeviceMemory ( BucketizedHashEntryInfo  hash_entry_info,
PerfectHashTableEntryInfo  hash_table_entry_info,
const size_t  shard_count,
const int  device_id,
const int  device_count,
const Executor executor 
)
inline

Definition at line 29 of file PerfectHashTableBuilder.h.

References BucketizedHashEntryInfo::bucketized_hash_entry_count, CHECK, CHECK_GT, get_entries_per_shard(), BucketizedHashEntryInfo::getNormalizedHashEntryCount(), GPU, hash_table_, HashTableEntryInfo::setNumHashEntries(), UNREACHABLE, and VLOG.

Referenced by PerfectJoinHashTable::copyCpuHashTableToGpu(), and PerfectJoinHashTable::initHashTableForDevice().

34  {
35 #ifdef HAVE_CUDA
36  if (shard_count) {
37  const auto shards_per_device = (shard_count + device_count - 1) / device_count;
38  CHECK_GT(shards_per_device, 0u);
39  const size_t entries_per_shard =
40  get_entries_per_shard(hash_entry_info.bucketized_hash_entry_count, shard_count);
41  hash_entry_info.bucketized_hash_entry_count = entries_per_shard * shards_per_device;
42  hash_table_entry_info.setNumHashEntries(
43  hash_entry_info.getNormalizedHashEntryCount());
44  }
46  hash_table_ = std::make_unique<PerfectHashTable>(ExecutorDeviceType::GPU,
47  hash_table_entry_info,
48  executor->getDataMgr(),
49  device_id);
50  if (hash_table_entry_info.getNumKeys() == 0) {
51  VLOG(1) << "Stop building a hash table based on a column: an input table is empty";
52  return;
53  }
54  hash_table_->allocateGpuMemory(hash_table_entry_info.computeTotalNumSlots());
55 #else
56  UNREACHABLE();
57 #endif // HAVE_CUDA
58  }
#define UNREACHABLE()
Definition: Logger.h:338
static size_t get_entries_per_shard(const size_t total_entry_count, const size_t shard_count)
#define CHECK_GT(x, y)
Definition: Logger.h:305
size_t getNormalizedHashEntryCount() const
#define CHECK(condition)
Definition: Logger.h:291
void setNumHashEntries(size_t num_hash_entries)
Definition: HashTable.h:54
std::unique_ptr< PerfectHashTable > hash_table_
size_t bucketized_hash_entry_count
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool PerfectJoinHashTableBuilder::for_semi_anti_join ( const JoinType  join_type)
inline

Definition at line 373 of file PerfectHashTableBuilder.h.

References ANTI, and SEMI.

Referenced by initOneToOneHashTableOnCpu().

373  {
374  return join_type == JoinType::SEMI || join_type == JoinType::ANTI;
375  }

+ Here is the caller graph for this function:

static size_t PerfectJoinHashTableBuilder::get_entries_per_shard ( const size_t  total_entry_count,
const size_t  shard_count 
)
inlinestatic

Definition at line 367 of file PerfectHashTableBuilder.h.

References CHECK_NE.

Referenced by allocateDeviceMemory().

368  {
369  CHECK_NE(size_t(0), shard_count);
370  return (total_entry_count + shard_count - 1) / shard_count;
371  }
#define CHECK_NE(x, y)
Definition: Logger.h:302

+ Here is the caller graph for this function:

std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::getHashTable ( )
inline

Definition at line 363 of file PerfectHashTableBuilder.h.

References hash_table_.

Referenced by PerfectJoinHashTable::copyCpuHashTableToGpu(), and PerfectJoinHashTable::initHashTableForDevice().

363  {
364  return std::move(hash_table_);
365  }
std::unique_ptr< PerfectHashTable > hash_table_

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToManyHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &  cols,
const StringDictionaryProxy::IdMap str_proxy_translation_map,
const JoinType  join_type,
const BucketizedHashEntryInfo  hash_entry_info,
const PerfectHashTableEntryInfo  hash_table_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 284 of file PerfectHashTableBuilder.h.

References run_benchmark_import::args, threading_serial::async(), CHECK, CPU, cpu_threads(), StringDictionaryProxy::TranslationMap< T >::data(), DEBUG_TIMER, StringDictionaryProxy::TranslationMap< T >::domainStart(), fill_one_to_many_hash_table(), fill_one_to_many_hash_table_bucketized(), get_join_column_type_kind(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), BucketizedHashEntryInfo::getNormalizedHashEntryCount(), HashTableEntryInfo::getNumKeys(), hash_table_, init_hash_join_buff(), inline_fixed_encoding_null_val(), kDATE, VLOG, and WINDOW_FUNCTION_FRAMING.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

294  {
295  auto timer = DEBUG_TIMER(__func__);
296  const auto inner_col = cols.first;
297  CHECK(inner_col);
298  const auto& ti = inner_col->get_type_info();
299  CHECK(!hash_table_);
300  hash_table_ = std::make_unique<PerfectHashTable>(ExecutorDeviceType::CPU,
301  hash_table_entry_info);
302  if (hash_table_entry_info.getNumKeys() == 0) {
303  VLOG(1) << "Stop building a hash table based on a column: an input table is empty";
304  return;
305  }
306  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
307  int thread_count = cpu_threads();
308  {
309  auto timer_init = DEBUG_TIMER("Initialize CPU One-To-Many Perfect Hash Table");
310 #ifdef HAVE_TBB
311  init_hash_join_buff_tbb(cpu_hash_table_buff,
312  hash_entry_info.getNormalizedHashEntryCount(),
313  hash_join_invalid_val);
314 #else // #ifdef HAVE_TBB
315  std::vector<std::future<void>> init_threads;
316  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
317  init_threads.emplace_back(
320  cpu_hash_table_buff,
321  hash_entry_info.getNormalizedHashEntryCount(),
322  hash_join_invalid_val,
323  thread_idx,
324  thread_count));
325  }
326  for (auto& child : init_threads) {
327  child.wait();
328  }
329  for (auto& child : init_threads) {
330  child.get();
331  }
332 #endif // !HAVE_TBB
333  }
334  auto timer_build = DEBUG_TIMER("Fill CPU One-To-Many Perfect Hash Table");
335  auto const use_bucketization = inner_col->get_type_info().get_type() == kDATE;
336  auto translated_null_val = col_range.getIntMax() + 1;
337  if (col_range.getIntMax() < col_range.getIntMin()) {
338  translated_null_val = col_range.getIntMin() - 1;
339  }
340  JoinColumnTypeInfo type_info{static_cast<size_t>(ti.get_size()),
341  col_range.getIntMin(),
342  col_range.getIntMax(),
344  is_bitwise_eq,
345  translated_null_val,
348  cpu_hash_table_buff,
349  hash_entry_info,
350  join_column,
351  type_info,
352  str_proxy_translation_map ? str_proxy_translation_map->data() : nullptr,
353  str_proxy_translation_map ? str_proxy_translation_map->domainStart()
354  : 0 /*dummy*/,
355  hash_entry_info.bucket_normalization,
356  join_type == JoinType::WINDOW_FUNCTION_FRAMING};
357  decltype(&fill_one_to_many_hash_table) const hash_table_fill_func =
358  use_bucketization ? fill_one_to_many_hash_table_bucketized
360  hash_table_fill_func(args, thread_count);
361  }
int64_t getIntMin() const
void fill_one_to_many_hash_table(OneToManyPerfectJoinHashTableFillFuncArgs const args, const int32_t cpu_thread_count)
future< Result > async(Fn &&fn, Args &&...args)
size_t getNumKeys() const
Definition: HashTable.h:51
Definition: sqltypes.h:80
size_t getNormalizedHashEntryCount() const
void fill_one_to_many_hash_table_bucketized(OneToManyPerfectJoinHashTableFillFuncArgs const args, const int32_t cpu_thread_count)
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:291
#define DEBUG_TIMER(name)
Definition: Logger.h:412
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::unique_ptr< PerfectHashTable > hash_table_
int cpu_threads()
Definition: thread_count.h:25
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToOneHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const InnerOuter cols,
const StringDictionaryProxy::IdMap str_proxy_translation_map,
const JoinType  join_type,
const BucketizedHashEntryInfo  hash_entry_info,
const PerfectHashTableEntryInfo  hash_table_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 186 of file PerfectHashTableBuilder.h.

References run_benchmark_import::args, threading_serial::async(), BucketizedHashEntryInfo::bucket_normalization, CHECK, CPU, cpu_threads(), StringDictionaryProxy::TranslationMap< T >::data(), DEBUG_TIMER, StringDictionaryProxy::TranslationMap< T >::domainStart(), fill_hash_join_buff(), fill_hash_join_buff_bitwise_eq(), fill_hash_join_buff_bucketized(), for_semi_anti_join(), get_join_column_type_kind(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), BucketizedHashEntryInfo::getNormalizedHashEntryCount(), HashTableEntryInfo::getNumKeys(), hash_table_, init_hash_join_buff(), inline_fixed_encoding_null_val(), kDATE, and VLOG.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

196  {
197  auto timer = DEBUG_TIMER(__func__);
198  const auto inner_col = cols.first;
199  CHECK(inner_col);
200  const auto& ti = inner_col->get_type_info();
201  CHECK(!hash_table_);
202  hash_table_ = std::make_unique<PerfectHashTable>(ExecutorDeviceType::CPU,
203  hash_table_entry_info);
204  if (hash_table_entry_info.getNumKeys() == 0) {
205  VLOG(1) << "Stop building a hash table based on a column: an input table is empty";
206  return;
207  }
208  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
209  const int thread_count = cpu_threads();
210  {
211  DEBUG_TIMER("Initialize CPU One-To-One Perfect Hash Table");
212 #ifdef HAVE_TBB
213  init_hash_join_buff_tbb(cpu_hash_table_buff,
214  hash_entry_info.getNormalizedHashEntryCount(),
215  hash_join_invalid_val);
216 #else // #ifdef HAVE_TBB
217  std::vector<std::thread> init_cpu_buff_threads;
218  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
219  init_cpu_buff_threads.emplace_back([hash_entry_info,
220  hash_join_invalid_val,
221  thread_idx,
222  thread_count,
223  cpu_hash_table_buff] {
224  init_hash_join_buff(cpu_hash_table_buff,
225  hash_entry_info.getNormalizedHashEntryCount(),
226  hash_join_invalid_val,
227  thread_idx,
228  thread_count);
229  });
230  }
231  for (auto& t : init_cpu_buff_threads) {
232  t.join();
233  }
234  init_cpu_buff_threads.clear();
235 #endif // !HAVE_TBB
236  }
237  auto const for_semi_join = for_semi_anti_join(join_type);
238  auto const use_bucketization = inner_col->get_type_info().get_type() == kDATE;
239  auto translated_null_val = col_range.getIntMax() + 1;
240  if (col_range.getIntMax() < col_range.getIntMin()) {
241  translated_null_val = col_range.getIntMin() - 1;
242  }
243  JoinColumnTypeInfo type_info{static_cast<size_t>(ti.get_size()),
244  col_range.getIntMin(),
245  col_range.getIntMax(),
247  is_bitwise_eq,
248  translated_null_val,
250  DEBUG_TIMER("Fill CPU One-To-One Perfect Hash Table");
252  cpu_hash_table_buff,
253  nullptr,
254  hash_join_invalid_val,
255  for_semi_join,
256  join_column,
257  type_info,
258  str_proxy_translation_map ? str_proxy_translation_map->data() : nullptr,
259  str_proxy_translation_map ? str_proxy_translation_map->domainStart()
260  : 0, // 0 is dummy value
261  hash_entry_info.bucket_normalization};
262  decltype(&fill_hash_join_buff) const hash_table_fill_func =
263  use_bucketization ? fill_hash_join_buff_bucketized
264  : type_info.uses_bw_eq ? fill_hash_join_buff_bitwise_eq
266 
267  std::vector<std::future<int>> fill_threads;
268  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
269  fill_threads.emplace_back(std::async(
270  std::launch::async, hash_table_fill_func, args, thread_idx, thread_count));
271  }
272  for (auto& child : fill_threads) {
273  child.wait();
274  }
275  for (auto& child : fill_threads) {
276  if (child.get()) { // see if task returns an error code
277  // Too many hash entries, need to retry with a 1:many table
278  hash_table_ = nullptr; // clear the hash table buffer
279  throw NeedsOneToManyHash();
280  }
281  }
282  }
int64_t getIntMin() const
DEVICE int SUFFIX() fill_hash_join_buff_bitwise_eq(OneToOnePerfectJoinHashTableFillFuncArgs const args, int32_t const cpu_thread_idx, int32_t const cpu_thread_count)
const bool for_semi_anti_join(const JoinType join_type)
DEVICE int SUFFIX() fill_hash_join_buff(OneToOnePerfectJoinHashTableFillFuncArgs const args, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
future< Result > async(Fn &&fn, Args &&...args)
int64_t bucket_normalization
size_t getNumKeys() const
Definition: HashTable.h:51
Definition: sqltypes.h:80
size_t getNormalizedHashEntryCount() const
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:291
#define DEBUG_TIMER(name)
Definition: Logger.h:412
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
DEVICE int SUFFIX() fill_hash_join_buff_bucketized(OneToOnePerfectJoinHashTableFillFuncArgs const args, int32_t const cpu_thread_idx, int32_t const cpu_thread_count)
std::unique_ptr< PerfectHashTable > hash_table_
int cpu_threads()
Definition: thread_count.h:25
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::hash_table_
private

The documentation for this class was generated from the following file: