OmniSciDB  94e8789169
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
PerfectJoinHashTableBuilder Class Reference

#include <PerfectHashTableBuilder.h>

+ Collaboration diagram for PerfectJoinHashTableBuilder:

Public Member Functions

 PerfectJoinHashTableBuilder (const Catalog_Namespace::Catalog *catalog)
 
void allocateDeviceMemory (const JoinColumn &join_column, const HashType layout, HashEntryInfo &hash_entry_info, const size_t shard_count, const int device_id, const int device_count)
 
void initOneToOneHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const InnerOuter &cols, const HashEntryInfo hash_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
void initOneToManyHashTableOnCpu (const JoinColumn &join_column, const ExpressionRange &col_range, const bool is_bitwise_eq, const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &cols, const HashEntryInfo hash_entry_info, const int32_t hash_join_invalid_val, const Executor *executor)
 
std::unique_ptr< PerfectHashTablegetHashTable ()
 

Static Public Member Functions

static size_t get_entries_per_shard (const size_t total_entry_count, const size_t shard_count)
 

Private Attributes

const Catalog_Namespace::Catalogcatalog_
 
std::unique_ptr< PerfectHashTablehash_table_
 

Detailed Description

Definition at line 23 of file PerfectHashTableBuilder.h.

Constructor & Destructor Documentation

PerfectJoinHashTableBuilder::PerfectJoinHashTableBuilder ( const Catalog_Namespace::Catalog catalog)
inline

Definition at line 25 of file PerfectHashTableBuilder.h.

26  : catalog_(catalog) {}
const Catalog_Namespace::Catalog * catalog_

Member Function Documentation

void PerfectJoinHashTableBuilder::allocateDeviceMemory ( const JoinColumn join_column,
const HashType  layout,
HashEntryInfo hash_entry_info,
const size_t  shard_count,
const int  device_id,
const int  device_count 
)
inline

Definition at line 28 of file PerfectHashTableBuilder.h.

References catalog_, CHECK, CHECK_GT, get_entries_per_shard(), HashEntryInfo::getNormalizedHashEntryCount(), GPU, HashEntryInfo::hash_entry_count, hash_table_, JoinColumn::num_elems, OneToOne, and UNREACHABLE.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

33  {
34 #ifdef HAVE_CUDA
35  if (shard_count) {
36  const auto shards_per_device = (shard_count + device_count - 1) / device_count;
37  CHECK_GT(shards_per_device, 0u);
38  const size_t entries_per_shard =
39  get_entries_per_shard(hash_entry_info.hash_entry_count, shard_count);
40  hash_entry_info.hash_entry_count = entries_per_shard * shards_per_device;
41  }
42  const size_t total_count =
43  layout == HashType::OneToOne
44  ? hash_entry_info.getNormalizedHashEntryCount()
45  : 2 * hash_entry_info.getNormalizedHashEntryCount() + join_column.num_elems;
47  hash_table_ =
48  std::make_unique<PerfectHashTable>(catalog_,
49  layout,
51  hash_entry_info.getNormalizedHashEntryCount(),
52  join_column.num_elems);
53  hash_table_->allocateGpuMemory(total_count, device_id);
54 #else
55  UNREACHABLE();
56 #endif // HAVE_CUDA
57  }
const Catalog_Namespace::Catalog * catalog_
size_t num_elems
#define UNREACHABLE()
Definition: Logger.h:241
static size_t get_entries_per_shard(const size_t total_entry_count, const size_t shard_count)
#define CHECK_GT(x, y)
Definition: Logger.h:209
size_t hash_entry_count
size_t getNormalizedHashEntryCount() const
#define CHECK(condition)
Definition: Logger.h:197
std::unique_ptr< PerfectHashTable > hash_table_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static size_t PerfectJoinHashTableBuilder::get_entries_per_shard ( const size_t  total_entry_count,
const size_t  shard_count 
)
inlinestatic

Definition at line 358 of file PerfectHashTableBuilder.h.

References CHECK_NE.

Referenced by allocateDeviceMemory().

359  {
360  CHECK_NE(size_t(0), shard_count);
361  return (total_entry_count + shard_count - 1) / shard_count;
362  }
#define CHECK_NE(x, y)
Definition: Logger.h:206

+ Here is the caller graph for this function:

std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::getHashTable ( )
inline

Definition at line 356 of file PerfectHashTableBuilder.h.

References hash_table_.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

356 { return std::move(hash_table_); }
std::unique_ptr< PerfectHashTable > hash_table_

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToManyHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > &  cols,
const HashEntryInfo  hash_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 270 of file PerfectHashTableBuilder.h.

References catalog_, CHECK, CHECK_EQ, CPU, cpu_threads(), DEBUG_TIMER, fill_one_to_many_hash_table(), fill_one_to_many_hash_table_bucketized(), get_join_column_type_kind(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), HashEntryInfo::getNormalizedHashEntryCount(), Executor::getStringDictionaryProxy(), hash_table_, init_hash_join_buff(), inline_fixed_encoding_null_val(), kDATE, kENCODING_DICT, JoinColumn::num_elems, and OneToMany.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

277  {
278  auto timer = DEBUG_TIMER(__func__);
279  const auto inner_col = cols.first;
280  CHECK(inner_col);
281  const auto& ti = inner_col->get_type_info();
282 
283  CHECK(!hash_table_);
284  hash_table_ =
285  std::make_unique<PerfectHashTable>(catalog_,
288  hash_entry_info.getNormalizedHashEntryCount(),
289  join_column.num_elems);
290 
291  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
292  const StringDictionaryProxy* sd_inner_proxy{nullptr};
293  const StringDictionaryProxy* sd_outer_proxy{nullptr};
294  if (ti.is_string()) {
295  CHECK_EQ(kENCODING_DICT, ti.get_compression());
296  sd_inner_proxy =
297  executor->getStringDictionaryProxy(inner_col->get_comp_param(), true);
298  CHECK(sd_inner_proxy);
299  const auto outer_col = dynamic_cast<const Analyzer::ColumnVar*>(cols.second);
300  CHECK(outer_col);
301  sd_outer_proxy =
302  executor->getStringDictionaryProxy(outer_col->get_comp_param(), true);
303  CHECK(sd_outer_proxy);
304  }
305  int thread_count = cpu_threads();
306  std::vector<std::future<void>> init_threads;
307  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
308  init_threads.emplace_back(std::async(std::launch::async,
310  cpu_hash_table_buff,
311  hash_entry_info.getNormalizedHashEntryCount(),
312  hash_join_invalid_val,
313  thread_idx,
314  thread_count));
315  }
316  for (auto& child : init_threads) {
317  child.wait();
318  }
319  for (auto& child : init_threads) {
320  child.get();
321  }
322 
323  if (ti.get_type() == kDATE) {
324  fill_one_to_many_hash_table_bucketized(cpu_hash_table_buff,
325  hash_entry_info,
326  hash_join_invalid_val,
327  join_column,
328  {static_cast<size_t>(ti.get_size()),
329  col_range.getIntMin(),
330  col_range.getIntMax(),
332  is_bitwise_eq,
333  col_range.getIntMax() + 1,
335  sd_inner_proxy,
336  sd_outer_proxy,
337  thread_count);
338  } else {
339  fill_one_to_many_hash_table(cpu_hash_table_buff,
340  hash_entry_info,
341  hash_join_invalid_val,
342  join_column,
343  {static_cast<size_t>(ti.get_size()),
344  col_range.getIntMin(),
345  col_range.getIntMax(),
347  is_bitwise_eq,
348  col_range.getIntMax() + 1,
350  sd_inner_proxy,
351  sd_outer_proxy,
352  thread_count);
353  }
354  }
int64_t getIntMin() const
#define CHECK_EQ(x, y)
Definition: Logger.h:205
const Catalog_Namespace::Catalog * catalog_
size_t num_elems
void fill_one_to_many_hash_table_bucketized(int32_t *buff, const HashEntryInfo hash_entry_info, const int32_t invalid_slot_val, const JoinColumn &join_column, const JoinColumnTypeInfo &type_info, const void *sd_inner_proxy, const void *sd_outer_proxy, const unsigned cpu_thread_count)
StringDictionaryProxy * getStringDictionaryProxy(const int dict_id, const bool with_generation) const
Definition: Execute.h:409
Definition: sqltypes.h:52
void fill_one_to_many_hash_table(int32_t *buff, const HashEntryInfo hash_entry_info, const int32_t invalid_slot_val, const JoinColumn &join_column, const JoinColumnTypeInfo &type_info, const void *sd_inner_proxy, const void *sd_outer_proxy, const unsigned cpu_thread_count)
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
size_t getNormalizedHashEntryCount() const
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::unique_ptr< PerfectHashTable > hash_table_
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PerfectJoinHashTableBuilder::initOneToOneHashTableOnCpu ( const JoinColumn join_column,
const ExpressionRange col_range,
const bool  is_bitwise_eq,
const InnerOuter cols,
const HashEntryInfo  hash_entry_info,
const int32_t  hash_join_invalid_val,
const Executor executor 
)
inline

Definition at line 172 of file PerfectHashTableBuilder.h.

References HashEntryInfo::bucket_normalization, catalog_, CHECK, CHECK_EQ, CPU, cpu_threads(), DEBUG_TIMER, fill_hash_join_buff_bucketized(), get_join_column_type_kind(), ExpressionRange::getIntMax(), ExpressionRange::getIntMin(), HashEntryInfo::getNormalizedHashEntryCount(), Executor::getStringDictionaryProxy(), hash_table_, init_hash_join_buff(), inline_fixed_encoding_null_val(), kENCODING_DICT, OneToOne, and generate_TableFunctionsFactory_init::t.

Referenced by PerfectJoinHashTable::initHashTableForDevice().

178  {
179  auto timer = DEBUG_TIMER(__func__);
180  const auto inner_col = cols.first;
181  CHECK(inner_col);
182  const auto& ti = inner_col->get_type_info();
183 
184  CHECK(!hash_table_);
185  hash_table_ =
186  std::make_unique<PerfectHashTable>(catalog_,
189  hash_entry_info.getNormalizedHashEntryCount(),
190  0);
191 
192  auto cpu_hash_table_buff = reinterpret_cast<int32_t*>(hash_table_->getCpuBuffer());
193  const StringDictionaryProxy* sd_inner_proxy{nullptr};
194  const StringDictionaryProxy* sd_outer_proxy{nullptr};
195  const auto outer_col = dynamic_cast<const Analyzer::ColumnVar*>(cols.second);
196  if (ti.is_string() &&
197  (outer_col && !(inner_col->get_comp_param() == outer_col->get_comp_param()))) {
198  CHECK_EQ(kENCODING_DICT, ti.get_compression());
199  sd_inner_proxy =
200  executor->getStringDictionaryProxy(inner_col->get_comp_param(), true);
201  CHECK(sd_inner_proxy);
202  CHECK(outer_col);
203  sd_outer_proxy =
204  executor->getStringDictionaryProxy(outer_col->get_comp_param(), true);
205  CHECK(sd_outer_proxy);
206  }
207  int thread_count = cpu_threads();
208  std::vector<std::thread> init_cpu_buff_threads;
209  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
210  init_cpu_buff_threads.emplace_back([hash_entry_info,
211  hash_join_invalid_val,
212  thread_idx,
213  thread_count,
214  cpu_hash_table_buff] {
215  init_hash_join_buff(cpu_hash_table_buff,
216  hash_entry_info.getNormalizedHashEntryCount(),
217  hash_join_invalid_val,
218  thread_idx,
219  thread_count);
220  });
221  }
222  for (auto& t : init_cpu_buff_threads) {
223  t.join();
224  }
225  init_cpu_buff_threads.clear();
226  std::atomic<int> err{0};
227  for (int thread_idx = 0; thread_idx < thread_count; ++thread_idx) {
228  init_cpu_buff_threads.emplace_back([hash_join_invalid_val,
229  &join_column,
230  sd_inner_proxy,
231  sd_outer_proxy,
232  thread_idx,
233  thread_count,
234  &ti,
235  &err,
236  &col_range,
237  &is_bitwise_eq,
238  cpu_hash_table_buff,
239  hash_entry_info] {
240  int partial_err =
241  fill_hash_join_buff_bucketized(cpu_hash_table_buff,
242  hash_join_invalid_val,
243  join_column,
244  {static_cast<size_t>(ti.get_size()),
245  col_range.getIntMin(),
246  col_range.getIntMax(),
248  is_bitwise_eq,
249  col_range.getIntMax() + 1,
251  sd_inner_proxy,
252  sd_outer_proxy,
253  thread_idx,
254  thread_count,
255  hash_entry_info.bucket_normalization);
256  int zero{0};
257  err.compare_exchange_strong(zero, partial_err);
258  });
259  }
260  for (auto& t : init_cpu_buff_threads) {
261  t.join();
262  }
263  if (err) {
264  // Too many hash entries, need to retry with a 1:many table
265  hash_table_ = nullptr; // clear the hash table buffer
266  throw NeedsOneToManyHash();
267  }
268  }
int64_t getIntMin() const
#define CHECK_EQ(x, y)
Definition: Logger.h:205
const Catalog_Namespace::Catalog * catalog_
StringDictionaryProxy * getStringDictionaryProxy(const int dict_id, const bool with_generation) const
Definition: Execute.h:409
int64_t bucket_normalization
DEVICE void SUFFIX() init_hash_join_buff(int32_t *groups_buffer, const int64_t hash_entry_count, const int32_t invalid_slot_val, const int32_t cpu_thread_idx, const int32_t cpu_thread_count)
size_t getNormalizedHashEntryCount() const
int64_t getIntMax() const
ColumnType get_join_column_type_kind(const SQLTypeInfo &ti)
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::unique_ptr< PerfectHashTable > hash_table_
DEVICE int SUFFIX() fill_hash_join_buff_bucketized(int32_t *buff, const int32_t invalid_slot_val, const JoinColumn join_column, const JoinColumnTypeInfo type_info, const void *sd_inner_proxy, const void *sd_outer_proxy, const int32_t cpu_thread_idx, const int32_t cpu_thread_count, const int64_t bucket_normalization)
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

const Catalog_Namespace::Catalog* PerfectJoinHashTableBuilder::catalog_
private
std::unique_ptr<PerfectHashTable> PerfectJoinHashTableBuilder::hash_table_
private

The documentation for this class was generated from the following file: