OmniSciDB  ba1bac9284
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
PerfectJoinHashTableBuilder Class Reference

#include <PerfectHashTableBuilder.h>

+ Collaboration diagram for PerfectJoinHashTableBuilder:

Public Member Functions

 PerfectJoinHashTableBuilder (const Catalog_Namespace::Catalog *catalog)
 
void allocateDeviceMemory (const JoinColumn &join_column, const HashType layout, HashEntryInfo &hash_entry_info, const size_t shard_count, const int device_id, const int device_count)
 
void initOneToOneHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const InnerOuter &cols, const JoinType join_type, const HashType hash_type, const HashEntryInfo hash_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
void initOneToManyHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &cols, const HashEntryInfo hash_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
std::unique_ptr< PerfectHashTablegetHashTable ()
 
const bool for_semi_anti_join (const JoinType join_type)
 

Static Public Member Functions

static size_t get_entries_per_shard (const size_t total_entry_count, const size_t shard_count)
 

Private Attributes

const Catalog_Namespace::Catalogcatalog_
 
std::unique_ptr< PerfectHashTablehash_table_
 

Detailed Description

Definition at line 23 of file PerfectHashTableBuilder.h.

Constructor & Destructor Documentation

PerfectJoinHashTableBuilder::PerfectJoinHashTableBuilder ( const Catalog_Namespace::Catalog catalog)
inline

Definition at line 25 of file PerfectHashTableBuilder.h.

26  : catalog_(catalog) {}
const Catalog_Namespace::Catalog * catalog_

Member Function Documentation

void PerfectJoinHashTableBuilder::allocateDeviceMemory ( const JoinColumn join_column,
const HashType  layout,
HashEntryInfo hash_entry_info,
const size_t  shard_count,
const int  device_id,
const int  device_count 
)
inline

Definition at line 28 of file PerfectHashTableBuilder.h.

References catalog_, CHECK, CHECK_GT, get_entries_per_shard(), HashEntryInfo::getNormalizedHashEntryCount(), GPU, HashEntryInfo::hash_entry_count, hash_table_, JoinColumn::num_elems, OneToOne, and UNREACHABLE.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

33  {
34 #ifdef HAVE_CUDA
35  if (shard_count) {
36  const auto shards_per_device = (shard_count + device_count - 1) / device_count;
37  CHECK_GT(shards_per_device, 0u);
38  const size_t entries_per_shard =
39  get_entries_per_shard(hash_entry_info.hash_entry_count, shard_count);
40  hash_entry_info.hash_entry_count = entries_per_shard * shards_per_device;
41  }
42  const size_t total_count =
43  layout == HashType::OneToOne
44  ? hash_entry_info.getNormalizedHashEntryCount()
45  : 2 * hash_entry_info.getNormalizedHashEntryCount() + join_column.num_elems;
47  hash_table_ =
48  std::make_unique<PerfectHashTable>(catalog_,
49  layout,
51  hash_entry_info.getNormalizedHashEntryCount(),
52  join_column.num_elems);
53  hash_table_->allocateGpuMemory(total_count, device_id);
54 #else
55  UNREACHABLE();
56 #endif // HAVE_CUDA
57  }
const Catalog_Namespace::Catalog * catalog_
size_t num_elems
#define UNREACHABLE()
Definition: Logger.h:250
static size_t get_entries_per_shard(const size_t total_entry_count, const size_t shard_count)
#define CHECK_GT(x, y)
Definition: Logger.h:218
size_t hash_entry_count
size_t getNormalizedHashEntryCount() const
#define CHECK(condition)
Definition: Logger.h:206
std::unique_ptr< PerfectHashTable > hash_table_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool PerfectJoinHashTableBuilder::for_semi_anti_join ( const JoinType  join_type)
inline

Definition at line 372 of file PerfectHashTableBuilder.h.

References ANTI, and SEMI.

Referenced by initOneToOneHashTableOnCpu().

372  {
373  return join_type == JoinType::SEMI || join_type == JoinType::ANTI;
374  }

+ Here is the caller graph for this function:

static size_t PerfectJoinHashTableBuilder::get_entries_per_shard ( const size_t  total_entry_count,
const size_t  shard_count 
)
inlinestatic

Definition at line 366 of file PerfectHashTableBuilder.h.

References CHECK_NE.

Referenced by allocateDeviceMemory().

367  {
368  CHECK_NE(size_t(0), shard_count);
369  return (total_entry_count + shard_count - 1) / shard_count;
370  }
#define CHECK_NE(x, y)
Definition: Logger.h:215

+ Here is the caller graph for this function:

std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::getHashTable ( )
inline

Definition at line 364 of file PerfectHashTableBuilder.h.

References hash_table_.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

364 { return std::move(hash_table_); }
std::unique_ptr< PerfectHashTable > hash_table_

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToManyHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &  cols,
const HashEntryInfo  hash_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 278 of file PerfectHashTableBuilder.h.

References catalog_, CHECK, CHECK_EQ, CPU, cpu_threads(), DEBUG_TIMER, fill_one_to_many_hash_table(), fill_one_to_many_hash_table_bucketized(), get_join_column_type_kind(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), HashEntryInfo::getNormalizedHashEntryCount(), Executor::getStringDictionaryProxy(), hash_table_, init_hash_join_buff(), inline_fixed_encoding_null_val(), kDATE, kENCODING_DICT, JoinColumn::num_elems, and OneToMany.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

285  {
286  auto timer = DEBUG_TIMER(__func__);
287  const auto inner_col = cols.first;
288  CHECK(inner_col);
289  const auto& ti = inner_col->get_type_info();
290 
291  CHECK(!hash_table_);
292  hash_table_ =
293  std::make_unique<PerfectHashTable>(catalog_,
296  hash_entry_info.getNormalizedHashEntryCount(),
297  join_column.num_elems);
298 
299  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
300  const StringDictionaryProxy* sd_inner_proxy{nullptr};
301  const StringDictionaryProxy* sd_outer_proxy{nullptr};
302  if (ti.is_string()) {
303  CHECK_EQ(kENCODING_DICT, ti.get_compression());
304  sd_inner_proxy =
305  executor->getStringDictionaryProxy(inner_col->get_comp_param(), true);
306  CHECK(sd_inner_proxy);
307  const auto outer_col = dynamic_cast<const Analyzer::ColumnVar*>(cols.second);
308  CHECK(outer_col);
309  sd_outer_proxy =
310  executor->getStringDictionaryProxy(outer_col->get_comp_param(), true);
311  CHECK(sd_outer_proxy);
312  }
313  int thread_count = cpu_threads();
314  std::vector<std::future<void>> init_threads;
315  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
316  init_threads.emplace_back(std::async(std::launch::async,
318  cpu_hash_table_buff,
319  hash_entry_info.getNormalizedHashEntryCount(),
320  hash_join_invalid_val,
321  thread_idx,
322  thread_count));
323  }
324  for (auto& child : init_threads) {
325  child.wait();
326  }
327  for (auto& child : init_threads) {
328  child.get();
329  }
330 
331  if (ti.get_type() == kDATE) {
332  fill_one_to_many_hash_table_bucketized(cpu_hash_table_buff,
333  hash_entry_info,
334  hash_join_invalid_val,
335  join_column,
336  {static_cast<size_t>(ti.get_size()),
337  col_range.getIntMin(),
338  col_range.getIntMax(),
340  is_bitwise_eq,
341  col_range.getIntMax() + 1,
343  sd_inner_proxy,
344  sd_outer_proxy,
345  thread_count);
346  } else {
347  fill_one_to_many_hash_table(cpu_hash_table_buff,
348  hash_entry_info,
349  hash_join_invalid_val,
350  join_column,
351  {static_cast<size_t>(ti.get_size()),
352  col_range.getIntMin(),
353  col_range.getIntMax(),
355  is_bitwise_eq,
356  col_range.getIntMax() + 1,
358  sd_inner_proxy,
359  sd_outer_proxy,
360  thread_count);
361  }
362  }
int64_t getIntMin() const
#define CHECK_EQ(x, y)
Definition: Logger.h:214
const Catalog_Namespace::Catalog * catalog_
size_t num_elems
void fill_one_to_many_hash_table_bucketized(int32_t *buff, const HashEntryInfo hash_entry_info, const int32_t invalid_slot_val, const JoinColumn &join_column, const JoinColumnTypeInfo &type_info, const void *sd_inner_proxy, const void *sd_outer_proxy, const unsigned cpu_thread_count)
StringDictionaryProxy * getStringDictionaryProxy(const int dict_id, const bool with_generation) const
Definition: Execute.h:400
Definition: sqltypes.h:52
void fill_one_to_many_hash_table(int32_t *buff, const HashEntryInfo hash_entry_info, const int32_t invalid_slot_val, const JoinColumn &join_column, const JoinColumnTypeInfo &type_info, const void *sd_inner_proxy, const void *sd_outer_proxy, const unsigned cpu_thread_count)
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
size_t getNormalizedHashEntryCount() const
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:206
#define DEBUG_TIMER(name)
Definition: Logger.h:322
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::unique_ptr< PerfectHashTable > hash_table_
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToOneHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const InnerOuter cols,
const JoinType  join_type,
const HashType  hash_type,
const HashEntryInfo  hash_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 175 of file PerfectHashTableBuilder.h.

References HashEntryInfo::bucket_normalization, catalog_, CHECK, CHECK_EQ, CPU, cpu_threads(), DEBUG_TIMER, fill_hash_join_buff_bucketized(), for_semi_anti_join(), get_join_column_type_kind(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), HashEntryInfo::getNormalizedHashEntryCount(), Executor::getStringDictionaryProxy(), hash_table_, init_hash_join_buff(), inline_fixed_encoding_null_val(), kENCODING_DICT, and t.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

183  {
184  auto timer = DEBUG_TIMER(__func__);
185  const auto inner_col = cols.first;
186  CHECK(inner_col);
187  const auto& ti = inner_col->get_type_info();
188 
189  CHECK(!hash_table_);
190  hash_table_ =
191  std::make_unique<PerfectHashTable>(catalog_,
192  hash_type,
194  hash_entry_info.getNormalizedHashEntryCount(),
195  0);
196 
197  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
198  const StringDictionaryProxy* sd_inner_proxy{nullptr};
199  const StringDictionaryProxy* sd_outer_proxy{nullptr};
200  const auto outer_col = dynamic_cast<const Analyzer::ColumnVar*>(cols.second);
201  const bool for_semi_join = for_semi_anti_join(join_type);
202  if (ti.is_string() &&
203  (outer_col && !(inner_col->get_comp_param() == outer_col->get_comp_param()))) {
204  CHECK_EQ(kENCODING_DICT, ti.get_compression());
205  sd_inner_proxy =
206  executor->getStringDictionaryProxy(inner_col->get_comp_param(), true);
207  CHECK(sd_inner_proxy);
208  CHECK(outer_col);
209  sd_outer_proxy =
210  executor->getStringDictionaryProxy(outer_col->get_comp_param(), true);
211  CHECK(sd_outer_proxy);
212  }
213  int thread_count = cpu_threads();
214  std::vector<std::thread> init_cpu_buff_threads;
215  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
216  init_cpu_buff_threads.emplace_back([hash_entry_info,
217  hash_join_invalid_val,
218  thread_idx,
219  thread_count,
220  cpu_hash_table_buff] {
221  init_hash_join_buff(cpu_hash_table_buff,
222  hash_entry_info.getNormalizedHashEntryCount(),
223  hash_join_invalid_val,
224  thread_idx,
225  thread_count);
226  });
227  }
228  for (auto& t : init_cpu_buff_threads) {
229  t.join();
230  }
231  init_cpu_buff_threads.clear();
232  std::atomic<int> err{0};
233  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
234  init_cpu_buff_threads.emplace_back([hash_join_invalid_val,
235  &join_column,
236  sd_inner_proxy,
237  sd_outer_proxy,
238  thread_idx,
239  thread_count,
240  &ti,
241  &err,
242  &col_range,
243  &is_bitwise_eq,
244  &for_semi_join,
245  cpu_hash_table_buff,
246  hash_entry_info] {
247  int partial_err =
248  fill_hash_join_buff_bucketized(cpu_hash_table_buff,
249  hash_join_invalid_val,
250  for_semi_join,
251  join_column,
252  {static_cast<size_t>(ti.get_size()),
253  col_range.getIntMin(),
254  col_range.getIntMax(),
256  is_bitwise_eq,
257  col_range.getIntMax() + 1,
259  sd_inner_proxy,
260  sd_outer_proxy,
261  thread_idx,
262  thread_count,
263  hash_entry_info.bucket_normalization);
264  int zero{0};
265  err.compare_exchange_strong(zero, partial_err);
266  });
267  }
268  for (auto& t : init_cpu_buff_threads) {
269  t.join();
270  }
271  if (err) {
272  // Too many hash entries, need to retry with a 1:many table
273  hash_table_ = nullptr; // clear the hash table buffer
274  throw NeedsOneToManyHash();
275  }
276  }
int64_t getIntMin() const
#define CHECK_EQ(x, y)
Definition: Logger.h:214
const Catalog_Namespace::Catalog * catalog_
const bool for_semi_anti_join(const JoinType join_type)
StringDictionaryProxy * getStringDictionaryProxy(const int dict_id, const bool with_generation) const
Definition: Execute.h:400
int64_t bucket_normalization
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
size_t getNormalizedHashEntryCount() const
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:206
#define DEBUG_TIMER(name)
Definition: Logger.h:322
char * t
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::unique_ptr< PerfectHashTable > hash_table_
int cpu_threads()
Definition: thread_count.h:24
DEVICE int SUFFIX() fill_hash_join_buff_bucketized(int32_t *buff, const int32_t invalid_slot_val, const bool for_semi_join, const JoinColumn join_column, const JoinColumnTypeInfo type_info, const void *sd_inner_proxy, const void *sd_outer_proxy, const int32_t cpu_thread_idx, const int32_t cpu_thread_count, const int64_t bucket_normalization)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

const Catalog_Namespace::Catalog* PerfectJoinHashTableBuilder::catalog_
private
std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::hash_table_
private

The documentation for this class was generated from the following file: