OmniSciDB  85c2d10cdc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
PerfectJoinHashTable.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file JoinHashTable.h
19  * @author Alex Suhan <alex@mapd.com>
20  *
21  */
22 
23 #pragma once
24 
25 #include "Analyzer/Analyzer.h"
26 #include "Catalog/Catalog.h"
28 #include "DataMgr/Chunk/Chunk.h"
37 
38 #include <llvm/IR/Value.h>
39 
40 #ifdef HAVE_CUDA
41 #include <cuda.h>
42 #endif
43 #include <functional>
44 #include <memory>
45 #include <mutex>
46 #include <stdexcept>
47 
48 struct HashEntryInfo;
49 
51  public:
52  using HashTableCacheValue = std::shared_ptr<PerfectHashTable>;
53 
55  static std::shared_ptr<PerfectJoinHashTable> getInstance(
56  const std::shared_ptr<Analyzer::BinOper> qual_bin_oper,
57  const std::vector<InputTableInfo>& query_infos,
58  const Data_Namespace::MemoryLevel memory_level,
59  const HashType preferred_hash_type,
60  const int device_count,
61  ColumnCacheMap& column_cache,
62  Executor* executor);
63 
64  std::string toString(const ExecutorDeviceType device_type,
65  const int device_id = 0,
66  bool raw = false) const override;
67 
68  std::set<DecodedJoinHashBufferEntry> toSet(const ExecutorDeviceType device_type,
69  const int device_id) const override;
70 
71  llvm::Value* codegenSlot(const CompilationOptions&, const size_t) override;
72 
74  const size_t) override;
75 
76  int getInnerTableId() const noexcept override {
77  return col_var_.get()->get_table_id();
78  };
79 
80  int getInnerTableRteIdx() const noexcept override {
81  return col_var_.get()->get_rte_idx();
82  };
83 
84  HashType getHashType() const noexcept override { return hash_type_; }
85 
87  return memory_level_;
88  };
89 
90  int getDeviceCount() const noexcept override { return device_count_; };
91 
92  size_t offsetBufferOff() const noexcept override;
93 
94  size_t countBufferOff() const noexcept override;
95 
96  size_t payloadBufferOff() const noexcept override;
97 
98  std::string getHashJoinType() const final { return "Perfect"; }
99 
100  static auto getHashTableCache() { return hash_table_cache_.get(); }
101 
102  static auto getCacheInvalidator() -> std::function<void()> {
104  return hash_table_cache_->getCacheInvalidator();
105  }
106 
108 
109  private:
110  // Equijoin API
112  const std::vector<Fragmenter_Namespace::FragmentInfo>& fragments,
113  const int device_id,
114  DeviceAllocator* dev_buff_owner);
115 
116  void reifyForDevice(const ChunkKey& hash_table_key,
117  const ColumnsForDevice& columns_for_device,
118  const HashType layout,
119  const int device_id,
120  const logger::ThreadId parent_thread_id);
121 
122  int initHashTableForDevice(const ChunkKey& chunk_key,
123  const JoinColumn& join_column,
124  const InnerOuter& cols,
125  const HashType layout,
126  const Data_Namespace::MemoryLevel effective_memory_level,
127  const int device_id);
128 
130  const std::vector<InnerOuter>& inner_outer_pairs) const;
131 
132  std::vector<InnerOuter> inner_outer_pairs_;
134 
135  PerfectJoinHashTable(const std::shared_ptr<Analyzer::BinOper> qual_bin_oper,
136  const Analyzer::ColumnVar* col_var,
137  const std::vector<InputTableInfo>& query_infos,
138  const Data_Namespace::MemoryLevel memory_level,
139  const HashType preferred_hash_type,
140  const ExpressionRange& col_range,
141  ColumnCacheMap& column_cache,
142  Executor* executor,
143  const int device_count)
144  : qual_bin_oper_(qual_bin_oper)
145  , col_var_(std::dynamic_pointer_cast<Analyzer::ColumnVar>(col_var->deep_copy()))
146  , query_infos_(query_infos)
147  , memory_level_(memory_level)
148  , hash_type_(preferred_hash_type)
149  , col_range_(col_range)
150  , executor_(executor)
151  , column_cache_(column_cache)
152  , device_count_(device_count) {
156  }
157 
159  const std::vector<Fragmenter_Namespace::FragmentInfo>& fragments,
160  const Analyzer::Expr* outer_col,
161  const Analyzer::ColumnVar* inner_col) const;
162 
163  void reify();
164  std::shared_ptr<PerfectHashTable> initHashTableOnCpuFromCache(const ChunkKey& chunk_key,
165  const size_t num_elements,
166  const InnerOuter& cols);
167  void putHashTableOnCpuToCache(const ChunkKey& chunk_key,
168  const size_t num_elements,
169  HashTableCacheValue hash_table,
170  const InnerOuter& cols);
171 
172  const InputTableInfo& getInnerQueryInfo(const Analyzer::ColumnVar* inner_col) const;
173 
174  size_t shardCount() const;
175 
176  llvm::Value* codegenHashTableLoad(const size_t table_idx);
177 
178  std::vector<llvm::Value*> getHashJoinArgs(llvm::Value* hash_ptr,
179  const Analyzer::Expr* key_col,
180  const int shard_count,
181  const CompilationOptions& co);
182 
183  bool isBitwiseEq() const;
184 
185  size_t getComponentBufferSize() const noexcept override;
186 
187  HashTable* getHashTableForDevice(const size_t device_id) const;
188 
189  std::shared_ptr<Analyzer::BinOper> qual_bin_oper_;
190  std::shared_ptr<Analyzer::ColumnVar> col_var_;
191  const std::vector<InputTableInfo>& query_infos_;
192  const Data_Namespace::MemoryLevel memory_level_;
194 
197  Executor* executor_;
199  const int device_count_;
200 
205  const size_t num_elements;
207  const SQLOps optype;
208 
209  bool operator==(const struct JoinHashTableCacheKey& that) const {
210  return col_range == that.col_range && inner_col == that.inner_col &&
211  outer_col == that.outer_col && num_elements == that.num_elements &&
212  chunk_key == that.chunk_key && optype == that.optype;
213  }
214  };
215 
216  static std::unique_ptr<HashTableCache<JoinHashTableCacheKey, HashTableCacheValue>>
218 };
219 
221  const Analyzer::Expr* outer_col,
222  const Executor* executor);
223 
224 std::vector<Fragmenter_Namespace::FragmentInfo> only_shards_for_device(
225  const std::vector<Fragmenter_Namespace::FragmentInfo>& fragments,
226  const int device_id,
227  const int device_count);
228 
230  const int inner_table_id,
231  const std::vector<InputTableInfo>& query_infos);
232 
233 size_t get_entries_per_device(const size_t total_entries,
234  const size_t shard_count,
235  const size_t device_count,
236  const Data_Namespace::MemoryLevel memory_level);
Defines data structures for the semantic analysis phase of query processing.
llvm::Value * codegenHashTableLoad(const size_t table_idx)
void reifyForDevice(const ChunkKey &hash_table_key, const ColumnsForDevice &columns_for_device, const HashType layout, const int device_id, const logger::ThreadId parent_thread_id)
std::vector< int > ChunkKey
Definition: types.h:37
Data_Namespace::MemoryLevel getEffectiveMemoryLevel(const std::vector< InnerOuter > &inner_outer_pairs) const
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:101
std::pair< const Analyzer::ColumnVar *, const Analyzer::Expr * > InnerOuter
Definition: HashJoin.h:76
const Data_Namespace::MemoryLevel memory_level_
size_t getComponentBufferSize() const noexceptoverride
ExecutorDeviceType
std::shared_ptr< Analyzer::BinOper > qual_bin_oper_
#define const
static auto getCacheInvalidator() -> std::function< void()>
SQLOps
Definition: sqldefs.h:29
size_t offsetBufferOff() const noexceptoverride
std::vector< std::shared_ptr< HashTable > > hash_tables_for_device_
Definition: HashJoin.h:269
static std::shared_ptr< PerfectJoinHashTable > getInstance(const std::shared_ptr< Analyzer::BinOper > qual_bin_oper, const std::vector< InputTableInfo > &query_infos, const Data_Namespace::MemoryLevel memory_level, const HashType preferred_hash_type, const int device_count, ColumnCacheMap &column_cache, Executor *executor)
Make hash table from an in-flight SQL query&#39;s parse tree etc.
int getDeviceCount() const noexceptoverride
const InputTableInfo & getInnerQueryInfo(const Analyzer::ColumnVar *inner_col) const
size_t payloadBufferOff() const noexceptoverride
ColumnsForDevice fetchColumnsForDevice(const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const int device_id, DeviceAllocator *dev_buff_owner)
bool needs_dictionary_translation(const Analyzer::ColumnVar *inner_col, const Analyzer::Expr *outer_col_expr, const Executor *executor)
#define CHECK_GT(x, y)
Definition: Logger.h:209
HashType getHashType() const noexceptoverride
std::vector< llvm::Value * > getHashJoinArgs(llvm::Value *hash_ptr, const Analyzer::Expr *key_col, const int shard_count, const CompilationOptions &co)
This file contains the class specification and related data structures for Catalog.
std::string getHashJoinType() const final
std::shared_ptr< PerfectHashTable > HashTableCacheValue
std::vector< Fragmenter_Namespace::FragmentInfo > only_shards_for_device(const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const int device_id, const int device_count)
std::shared_ptr< PerfectHashTable > initHashTableOnCpuFromCache(const ChunkKey &chunk_key, const size_t num_elements, const InnerOuter &cols)
PerfectJoinHashTable(const std::shared_ptr< Analyzer::BinOper > qual_bin_oper, const Analyzer::ColumnVar *col_var, const std::vector< InputTableInfo > &query_infos, const Data_Namespace::MemoryLevel memory_level, const HashType preferred_hash_type, const ExpressionRange &col_range, ColumnCacheMap &column_cache, Executor *executor, const int device_count)
static std::unique_ptr< HashTableCache< JoinHashTableCacheKey, HashTableCacheValue > > hash_table_cache_
std::vector< InnerOuter > inner_outer_pairs_
std::unordered_map< int, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
bool operator==(const struct JoinHashTableCacheKey &that) const
const InputTableInfo & get_inner_query_info(const int inner_table_id, const std::vector< InputTableInfo > &query_infos)
const std::vector< InputTableInfo > & query_infos_
int getInnerTableId() const noexceptoverride
std::string toString(const ExecutorDeviceType device_type, const int device_id=0, bool raw=false) const override
ColumnCacheMap & column_cache_
size_t get_entries_per_device(const size_t total_entries, const size_t shard_count, const size_t device_count, const Data_Namespace::MemoryLevel memory_level)
std::shared_ptr< Analyzer::ColumnVar > col_var_
uint64_t ThreadId
Definition: Logger.h:306
llvm::Value * codegenSlot(const CompilationOptions &, const size_t) override
ExpressionRangeType getType() const
Catalog_Namespace::Catalog * catalog_
#define CHECK(condition)
Definition: Logger.h:197
size_t countBufferOff() const noexceptoverride
int initHashTableForDevice(const ChunkKey &chunk_key, const JoinColumn &join_column, const InnerOuter &cols, const HashType layout, const Data_Namespace::MemoryLevel effective_memory_level, const int device_id)
ChunkKey genHashTableKey(const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const Analyzer::Expr *outer_col, const Analyzer::ColumnVar *inner_col) const
HashTable * getHashTableForDevice(const size_t device_id) const
Data_Namespace::MemoryLevel getMemoryLevel() const noexceptoverride
std::set< DecodedJoinHashBufferEntry > toSet(const ExecutorDeviceType device_type, const int device_id) const override
HashType
Definition: HashTable.h:19
void putHashTableOnCpuToCache(const ChunkKey &chunk_key, const size_t num_elements, HashTableCacheValue hash_table, const InnerOuter &cols)
HashJoinMatchingSet codegenMatchingSet(const CompilationOptions &, const size_t) override
int getInnerTableRteIdx() const noexceptoverride