OmniSciDB  85c2d10cdc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ColumnFetcher.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
23 
24 namespace std {
25 template <>
26 struct hash<std::vector<int>> {
27  size_t operator()(const std::vector<int>& vec) const {
28  return vec.size() ^ boost::hash_range(vec.begin(), vec.end());
29  }
30 };
31 
32 template <>
33 struct hash<std::pair<int, int>> {
34  size_t operator()(const std::pair<int, int>& p) const {
35  return boost::hash<std::pair<int, int>>()(p);
36  }
37 };
38 
39 } // namespace std
40 
41 struct FetchResult {
42  std::vector<std::vector<const int8_t*>> col_buffers;
43  std::vector<std::vector<int64_t>> num_rows;
44  std::vector<std::vector<uint64_t>> frag_offsets;
45 };
46 
48  public:
49  ColumnFetcher(Executor* executor, const ColumnCacheMap& column_cache);
50 
52  static std::pair<const int8_t*, size_t> getOneColumnFragment(
53  Executor* executor,
54  const Analyzer::ColumnVar& hash_col,
55  const Fragmenter_Namespace::FragmentInfo& fragment,
56  const Data_Namespace::MemoryLevel effective_mem_lvl,
57  const int device_id,
58  DeviceAllocator* device_allocator,
59  const size_t thread_idx,
60  std::vector<std::shared_ptr<Chunk_NS::Chunk>>& chunks_owner,
61  ColumnCacheMap& column_cache);
62 
65  Executor* executor,
66  const Analyzer::ColumnVar& hash_col,
67  const std::vector<Fragmenter_Namespace::FragmentInfo>& fragments,
68  const Data_Namespace::MemoryLevel effective_mem_lvl,
69  const int device_id,
70  DeviceAllocator* device_allocator,
71  const size_t thread_idx,
72  std::vector<std::shared_ptr<Chunk_NS::Chunk>>& chunks_owner,
73  std::vector<std::shared_ptr<void>>& malloc_owner,
74  ColumnCacheMap& column_cache);
75 
76  const int8_t* getOneTableColumnFragment(
77  const int table_id,
78  const int frag_id,
79  const int col_id,
80  const std::map<int, const TableFragments*>& all_tables_fragments,
81  std::list<std::shared_ptr<Chunk_NS::Chunk>>& chunk_holder,
82  std::list<ChunkIter>& chunk_iter_holder,
83  const Data_Namespace::MemoryLevel memory_level,
84  const int device_id,
85  DeviceAllocator* device_allocator) const;
86 
87  const int8_t* getAllTableColumnFragments(
88  const int table_id,
89  const int col_id,
90  const std::map<int, const TableFragments*>& all_tables_fragments,
91  const Data_Namespace::MemoryLevel memory_level,
92  const int device_id,
93  DeviceAllocator* device_allocator,
94  const size_t thread_idx) const;
95 
96  const int8_t* getResultSetColumn(const InputColDescriptor* col_desc,
97  const Data_Namespace::MemoryLevel memory_level,
98  const int device_id,
99  DeviceAllocator* device_allocator,
100  const size_t thread_idx) const;
101 
102  const int8_t* linearizeColumnFragments(
103  const int table_id,
104  const int col_id,
105  const std::map<int, const TableFragments*>& all_tables_fragments,
106  std::list<std::shared_ptr<Chunk_NS::Chunk>>& chunk_holder,
107  std::list<ChunkIter>& chunk_iter_holder,
108  const Data_Namespace::MemoryLevel memory_level,
109  const int device_id,
110  DeviceAllocator* device_allocator,
111  const size_t thread_idx) const;
112 
113  private:
114  static const int8_t* transferColumnIfNeeded(
115  const ColumnarResults* columnar_results,
116  const int col_id,
117  Data_Namespace::DataMgr* data_mgr,
118  const Data_Namespace::MemoryLevel memory_level,
119  const int device_id,
120  DeviceAllocator* device_allocator);
121 
122  void addMergedChunk(const InputColDescriptor col_desc,
123  const int device_id,
124  std::shared_ptr<Chunk_NS::Chunk> chunk_ptr,
125  int8_t* chunk_iter_ptr) const;
126 
127  const int8_t* getChunkiter(const InputColDescriptor col_desc,
128  const int device_id = 0) const;
129 
131  ChunkIter& chunk_iter,
132  const size_t total_num_tuples) const;
133 
134  const int8_t* getResultSetColumn(const ResultSetPtr& buffer,
135  const int table_id,
136  const int col_id,
137  const Data_Namespace::MemoryLevel memory_level,
138  const int device_id,
139  DeviceAllocator* device_allocator,
140  const size_t thread_idx) const;
141 
142  Executor* executor_;
143  using CacheKey = std::vector<int>;
144  mutable std::mutex columnar_fetch_mutex_;
146  mutable std::unordered_map<
148  std::unordered_map<CacheKey, std::unique_ptr<const ColumnarResults>>>
150  mutable std::unordered_map<InputColDescriptor, std::unique_ptr<const ColumnarResults>>
152  using DeviceMergedChunkMap = std::unordered_map<int, std::shared_ptr<Chunk_NS::Chunk>>;
153  mutable std::unordered_map<InputColDescriptor, DeviceMergedChunkMap>
155  using DeviceMergedChunkIterMap = std::unordered_map<int, int8_t*>;
156  mutable std::unordered_map<InputColDescriptor, DeviceMergedChunkIterMap>
158 
160  friend class TableFunctionExecutionContext; // TODO(adb)
161 };
std::unordered_map< int, std::shared_ptr< Chunk_NS::Chunk >> DeviceMergedChunkMap
std::unordered_map< InputColDescriptor, std::unordered_map< CacheKey, std::unique_ptr< const ColumnarResults > > > columnarized_ref_table_cache_
std::unordered_map< int, int8_t * > DeviceMergedChunkIterMap
static JoinColumn makeJoinColumn(Executor *executor, const Analyzer::ColumnVar &hash_col, const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, std::vector< std::shared_ptr< void >> &malloc_owner, ColumnCacheMap &column_cache)
Creates a JoinColumn struct containing an array of JoinChunk structs.
std::mutex columnar_fetch_mutex_
const int8_t * getResultSetColumn(const InputColDescriptor *col_desc, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx) const
size_t operator()(const std::vector< int > &vec) const
Definition: ColumnFetcher.h:27
ColumnCacheMap columnarized_table_cache_
const int8_t * linearizeColumnFragments(const int table_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunk_holder, std::list< ChunkIter > &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx) const
std::shared_ptr< ResultSet > ResultSetPtr
const int8_t * getChunkiter(const InputColDescriptor col_desc, const int device_id=0) const
Executor * executor_
void addMergedChunk(const InputColDescriptor col_desc, const int device_id, std::shared_ptr< Chunk_NS::Chunk > chunk_ptr, int8_t *chunk_iter_ptr) const
std::unordered_map< InputColDescriptor, std::unique_ptr< const ColumnarResults > > columnarized_scan_table_cache_
const int8_t * getOneTableColumnFragment(const int table_id, const int frag_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunk_holder, std::list< ChunkIter > &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(r...
Definition: Fragmenter.h:77
An AbstractBuffer is a unit of data management for a data manager.
ColumnFetcher(Executor *executor, const ColumnCacheMap &column_cache)
std::unordered_map< int, std::unordered_map< int, std::shared_ptr< const ColumnarResults >>> ColumnCacheMap
size_t operator()(const std::pair< int, int > &p) const
Definition: ColumnFetcher.h:34
static const int8_t * transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
static std::pair< const int8_t *, size_t > getOneColumnFragment(Executor *executor, const Analyzer::ColumnVar &hash_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, ColumnCacheMap &column_cache)
Gets one chunk&#39;s pointer and element count on either CPU or GPU.
Abstract class for managing device memory allocations.
std::vector< int > CacheKey
std::vector< std::vector< const int8_t * > > col_buffers
Definition: ColumnFetcher.h:42
std::vector< std::vector< int64_t > > num_rows
Definition: ColumnFetcher.h:43
std::unordered_map< InputColDescriptor, DeviceMergedChunkIterMap > linearized_multi_frag_chunk_iter_cache_
std::vector< std::vector< uint64_t > > frag_offsets
Definition: ColumnFetcher.h:44
ChunkIter prepareChunkIter(AbstractBuffer *merged, ChunkIter &chunk_iter, const size_t total_num_tuples) const
Descriptor for the fragments required for an execution kernel.
const int8_t * getAllTableColumnFragments(const int table_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator, const size_t thread_idx) const
std::unordered_map< InputColDescriptor, DeviceMergedChunkMap > linearized_multi_frag_table_cache_