OmniSciDB  94e8789169
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ColumnFetcher Class Reference

#include <ColumnFetcher.h>

+ Collaboration diagram for ColumnFetcher:

Public Member Functions

 ColumnFetcher (Executor *executor, const ColumnCacheMap &column_cache)
 
const int8_t * getOneTableColumnFragment (const int table_id, const int frag_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunk_holder, std::list< ChunkIter > &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 
const int8_t * getAllTableColumnFragments (const int table_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 
const int8_t * getResultSetColumn (const InputColDescriptor *col_desc, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 

Static Public Member Functions

static std::pair< const int8_t
*, size_t > 
getOneColumnFragment (Executor *executor, const Analyzer::ColumnVar &hash_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, ColumnCacheMap &column_cache)
 Gets one chunk's pointer and element count on either CPU or GPU. More...
 
static JoinColumn makeJoinColumn (Executor *executor, const Analyzer::ColumnVar &hash_col, const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, std::vector< std::shared_ptr< void >> &malloc_owner, ColumnCacheMap &column_cache)
 Creates a JoinColumn struct containing an array of JoinChunk structs. More...
 

Private Types

using CacheKey = std::vector< int >
 

Private Member Functions

const int8_t * getResultSetColumn (const ResultSetPtr &buffer, const int table_id, const int col_id, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 

Static Private Member Functions

static const int8_t * transferColumnIfNeeded (const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
 

Private Attributes

Executorexecutor_
 
std::mutex columnar_conversion_mutex_
 
ColumnCacheMap columnarized_table_cache_
 
std::unordered_map
< InputColDescriptor,
std::unordered_map< CacheKey,
std::unique_ptr< const
ColumnarResults > > > 
columnarized_ref_table_cache_
 
std::unordered_map
< InputColDescriptor,
std::unique_ptr< const
ColumnarResults > > 
columnarized_scan_table_cache_
 

Friends

class QueryCompilationDescriptor
 
class TableFunctionExecutionContext
 

Detailed Description

Definition at line 47 of file ColumnFetcher.h.

Member Typedef Documentation

using ColumnFetcher::CacheKey = std::vector<int>
private

Definition at line 115 of file ColumnFetcher.h.

Constructor & Destructor Documentation

ColumnFetcher::ColumnFetcher ( Executor executor,
const ColumnCacheMap column_cache 
)

Definition at line 23 of file ColumnFetcher.cpp.

24  : executor_(executor), columnarized_table_cache_(column_cache) {}
ColumnCacheMap columnarized_table_cache_
Executor * executor_

Member Function Documentation

const int8_t * ColumnFetcher::getAllTableColumnFragments ( const int  table_id,
const int  col_id,
const std::map< int, const TableFragments * > &  all_tables_fragments,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const

Definition at line 234 of file ColumnFetcher.cpp.

References CHECK, columnar_conversion_mutex_, columnarized_scan_table_cache_, Data_Namespace::CPU_LEVEL, executor_, getOneTableColumnFragment(), InputColDescriptor::getScanDesc(), InputDescriptor::getSourceType(), ColumnarResults::mergeResults(), TABLE, and transferColumnIfNeeded().

Referenced by Executor::fetchChunks(), and Executor::fetchUnionChunks().

240  {
241  const auto fragments_it = all_tables_fragments.find(table_id);
242  CHECK(fragments_it != all_tables_fragments.end());
243  const auto fragments = fragments_it->second;
244  const auto frag_count = fragments->size();
245  std::vector<std::unique_ptr<ColumnarResults>> column_frags;
246  const ColumnarResults* table_column = nullptr;
247  const InputColDescriptor col_desc(col_id, table_id, int(0));
248  CHECK(col_desc.getScanDesc().getSourceType() == InputSourceType::TABLE);
249  {
250  std::lock_guard<std::mutex> columnar_conversion_guard(columnar_conversion_mutex_);
251  auto column_it = columnarized_scan_table_cache_.find(col_desc);
252  if (column_it == columnarized_scan_table_cache_.end()) {
253  for (size_t frag_id = 0; frag_id < frag_count; ++frag_id) {
254  std::list<std::shared_ptr<Chunk_NS::Chunk>> chunk_holder;
255  std::list<ChunkIter> chunk_iter_holder;
256  const auto& fragment = (*fragments)[frag_id];
257  if (fragment.isEmptyPhysicalFragment()) {
258  continue;
259  }
260  auto chunk_meta_it = fragment.getChunkMetadataMap().find(col_id);
261  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
262  auto col_buffer = getOneTableColumnFragment(table_id,
263  static_cast<int>(frag_id),
264  col_id,
265  all_tables_fragments,
266  chunk_holder,
267  chunk_iter_holder,
269  int(0),
270  device_allocator);
271  column_frags.push_back(
272  std::make_unique<ColumnarResults>(executor_->row_set_mem_owner_,
273  col_buffer,
274  fragment.getNumTuples(),
275  chunk_meta_it->second->sqlType));
276  }
277  auto merged_results =
278  ColumnarResults::mergeResults(executor_->row_set_mem_owner_, column_frags);
279  table_column = merged_results.get();
280  columnarized_scan_table_cache_.emplace(col_desc, std::move(merged_results));
281  } else {
282  table_column = column_it->second.get();
283  }
284  }
285  return ColumnFetcher::transferColumnIfNeeded(table_column,
286  0,
287  &executor_->getCatalog()->getDataMgr(),
288  memory_level,
289  device_id,
290  device_allocator);
291 }
static std::unique_ptr< ColumnarResults > mergeResults(const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const std::vector< std::unique_ptr< ColumnarResults >> &sub_results)
Executor * executor_
std::unordered_map< InputColDescriptor, std::unique_ptr< const ColumnarResults > > columnarized_scan_table_cache_
const int8_t * getOneTableColumnFragment(const int table_id, const int frag_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunk_holder, std::list< ChunkIter > &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
std::mutex columnar_conversion_mutex_
static const int8_t * transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair< const int8_t *, size_t > ColumnFetcher::getOneColumnFragment ( Executor executor,
const Analyzer::ColumnVar hash_col,
const Fragmenter_Namespace::FragmentInfo fragment,
const Data_Namespace::MemoryLevel  effective_mem_lvl,
const int  device_id,
DeviceAllocator device_allocator,
std::vector< std::shared_ptr< Chunk_NS::Chunk >> &  chunks_owner,
ColumnCacheMap column_cache 
)
static

Gets one chunk's pointer and element count on either CPU or GPU.

Gets a column fragment chunk on CPU or on GPU depending on the effective memory level parameter. For temporary tables, the chunk will be copied to the GPU if needed. Returns a buffer pointer and an element count.

Definition at line 29 of file ColumnFetcher.cpp.

References CHECK, columnarize_result(), Data_Namespace::CPU_LEVEL, Fragmenter_Namespace::FragmentInfo::fragmentId, get_column_descriptor_maybe(), Analyzer::ColumnVar::get_column_id(), Analyzer::ColumnVar::get_table_id(), get_temporary_table(), Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMap(), Fragmenter_Namespace::FragmentInfo::getNumTuples(), Fragmenter_Namespace::FragmentInfo::isEmptyPhysicalFragment(), Fragmenter_Namespace::FragmentInfo::physicalTableId, and transferColumnIfNeeded().

Referenced by RelAlgExecutor::createWindowFunctionContext(), TableFunctionExecutionContext::execute(), and makeJoinColumn().

37  {
38  static std::mutex columnar_conversion_mutex;
39  if (fragment.isEmptyPhysicalFragment()) {
40  return {nullptr, 0};
41  }
42  const auto table_id = hash_col.get_table_id();
43  auto chunk_meta_it = fragment.getChunkMetadataMap().find(hash_col.get_column_id());
44  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
45  const auto& catalog = *executor->getCatalog();
46  const auto cd =
47  get_column_descriptor_maybe(hash_col.get_column_id(), table_id, catalog);
48  CHECK(!cd || !(cd->isVirtualCol));
49  const int8_t* col_buff = nullptr;
50  if (cd) { // real table
51  ChunkKey chunk_key{catalog.getCurrentDB().dbId,
52  fragment.physicalTableId,
53  hash_col.get_column_id(),
54  fragment.fragmentId};
55  const auto chunk = Chunk_NS::Chunk::getChunk(
56  cd,
57  &catalog.getDataMgr(),
58  chunk_key,
59  effective_mem_lvl,
60  effective_mem_lvl == Data_Namespace::CPU_LEVEL ? 0 : device_id,
61  chunk_meta_it->second->numBytes,
62  chunk_meta_it->second->numElements);
63  chunks_owner.push_back(chunk);
64  CHECK(chunk);
65  auto ab = chunk->getBuffer();
66  CHECK(ab->getMemoryPtr());
67  col_buff = reinterpret_cast<int8_t*>(ab->getMemoryPtr());
68  } else { // temporary table
69  const ColumnarResults* col_frag{nullptr};
70  {
71  std::lock_guard<std::mutex> columnar_conversion_guard(columnar_conversion_mutex);
72  const auto frag_id = fragment.fragmentId;
73  if (column_cache.empty() || !column_cache.count(table_id)) {
74  column_cache.insert(std::make_pair(
75  table_id, std::unordered_map<int, std::shared_ptr<const ColumnarResults>>()));
76  }
77  auto& frag_id_to_result = column_cache[table_id];
78  if (frag_id_to_result.empty() || !frag_id_to_result.count(frag_id)) {
79  frag_id_to_result.insert(
80  std::make_pair(frag_id,
81  std::shared_ptr<const ColumnarResults>(columnarize_result(
82  executor->row_set_mem_owner_,
83  get_temporary_table(executor->temporary_tables_, table_id),
84  frag_id))));
85  }
86  col_frag = column_cache[table_id][frag_id].get();
87  }
88  col_buff = transferColumnIfNeeded(
89  col_frag,
90  hash_col.get_column_id(),
91  &catalog.getDataMgr(),
92  effective_mem_lvl,
93  effective_mem_lvl == Data_Namespace::CPU_LEVEL ? 0 : device_id,
94  device_allocator);
95  }
96  return {col_buff, fragment.getNumTuples()};
97 }
int get_table_id() const
Definition: Analyzer.h:194
std::vector< int > ChunkKey
Definition: types.h:37
const ResultSetPtr & get_temporary_table(const TemporaryTables *temporary_tables, const int table_id)
Definition: Execute.h:224
const ColumnDescriptor * get_column_descriptor_maybe(const int col_id, const int table_id, const Catalog_Namespace::Catalog &cat)
Definition: Execute.h:216
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
const ChunkMetadataMap & getChunkMetadataMap() const
static const int8_t * transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
const ColumnarResults * columnarize_result(std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const ResultSetPtr &result, const int frag_id)
Definition: Execute.h:268
#define CHECK(condition)
Definition: Logger.h:197
int get_column_id() const
Definition: Analyzer.h:195

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::getOneTableColumnFragment ( const int  table_id,
const int  frag_id,
const int  col_id,
const std::map< int, const TableFragments * > &  all_tables_fragments,
std::list< std::shared_ptr< Chunk_NS::Chunk >> &  chunk_holder,
std::list< ChunkIter > &  chunk_iter_holder,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const

Definition at line 157 of file ColumnFetcher.cpp.

References Allocator::alloc(), cat(), CHECK, CHECK_EQ, CHECK_GT, DeviceAllocator::copyToDevice(), Data_Namespace::CPU_LEVEL, executor_, get_column_descriptor(), get_column_type(), Chunk_NS::Chunk::getChunk(), Data_Namespace::GPU_LEVEL, and kENCODING_NONE.

Referenced by Executor::fetchChunks(), Executor::fetchUnionChunks(), and getAllTableColumnFragments().

166  {
167  static std::mutex varlen_chunk_mutex; // TODO(alex): remove
168  static std::mutex chunk_list_mutex;
169  const auto fragments_it = all_tables_fragments.find(table_id);
170  CHECK(fragments_it != all_tables_fragments.end());
171  const auto fragments = fragments_it->second;
172  const auto& fragment = (*fragments)[frag_id];
173  if (fragment.isEmptyPhysicalFragment()) {
174  return nullptr;
175  }
176  std::shared_ptr<Chunk_NS::Chunk> chunk;
177  auto chunk_meta_it = fragment.getChunkMetadataMap().find(col_id);
178  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
179  CHECK(table_id > 0);
180  const auto& cat = *executor_->getCatalog();
181  auto cd = get_column_descriptor(col_id, table_id, cat);
182  CHECK(cd);
183  const auto col_type =
184  get_column_type(col_id, table_id, cd, executor_->temporary_tables_);
185  const bool is_real_string =
186  col_type.is_string() && col_type.get_compression() == kENCODING_NONE;
187  const bool is_varlen =
188  is_real_string ||
189  col_type.is_array(); // TODO: should it be col_type.is_varlen_array() ?
190  {
191  ChunkKey chunk_key{
192  cat.getCurrentDB().dbId, fragment.physicalTableId, col_id, fragment.fragmentId};
193  std::unique_ptr<std::lock_guard<std::mutex>> varlen_chunk_lock;
194  if (is_varlen) {
195  varlen_chunk_lock.reset(new std::lock_guard<std::mutex>(varlen_chunk_mutex));
196  }
198  cd,
199  &cat.getDataMgr(),
200  chunk_key,
201  memory_level,
202  memory_level == Data_Namespace::CPU_LEVEL ? 0 : device_id,
203  chunk_meta_it->second->numBytes,
204  chunk_meta_it->second->numElements);
205  std::lock_guard<std::mutex> chunk_list_lock(chunk_list_mutex);
206  chunk_holder.push_back(chunk);
207  }
208  if (is_varlen) {
209  CHECK_GT(table_id, 0);
210  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
211  chunk_iter_holder.push_back(chunk->begin_iterator(chunk_meta_it->second));
212  auto& chunk_iter = chunk_iter_holder.back();
213  if (memory_level == Data_Namespace::CPU_LEVEL) {
214  return reinterpret_cast<int8_t*>(&chunk_iter);
215  } else {
216  auto ab = chunk->getBuffer();
217  ab->pin();
218  auto& row_set_mem_owner = executor_->getRowSetMemoryOwner();
219  row_set_mem_owner->addVarlenInputBuffer(ab);
220  CHECK_EQ(Data_Namespace::GPU_LEVEL, memory_level);
221  CHECK(allocator);
222  auto chunk_iter_gpu = allocator->alloc(sizeof(ChunkIter));
223  allocator->copyToDevice(
224  chunk_iter_gpu, reinterpret_cast<int8_t*>(&chunk_iter), sizeof(ChunkIter));
225  return chunk_iter_gpu;
226  }
227  } else {
228  auto ab = chunk->getBuffer();
229  CHECK(ab->getMemoryPtr());
230  return ab->getMemoryPtr(); // @TODO(alex) change to use ChunkIter
231  }
232 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::vector< int > ChunkKey
Definition: types.h:37
std::string cat(Ts &&...args)
const SQLTypeInfo get_column_type(const int col_id, const int table_id, const ColumnDescriptor *cd, const TemporaryTables *temporary_tables)
Definition: Execute.h:232
Executor * executor_
#define CHECK_GT(x, y)
Definition: Logger.h:209
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
#define CHECK(condition)
Definition: Logger.h:197
const ColumnDescriptor * get_column_descriptor(const int col_id, const int table_id, const Catalog_Namespace::Catalog &cat)
Definition: Execute.h:187

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::getResultSetColumn ( const InputColDescriptor col_desc,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const

Definition at line 293 of file ColumnFetcher.cpp.

References CHECK, executor_, get_temporary_table(), InputColDescriptor::getColId(), InputColDescriptor::getScanDesc(), and InputDescriptor::getTableId().

Referenced by Executor::fetchChunks(), and Executor::fetchUnionChunks().

297  {
298  CHECK(col_desc);
299  const auto table_id = col_desc->getScanDesc().getTableId();
300  return getResultSetColumn(get_temporary_table(executor_->temporary_tables_, table_id),
301  table_id,
302  col_desc->getColId(),
303  memory_level,
304  device_id,
305  device_allocator);
306 }
Executor * executor_
const ResultSetPtr & get_temporary_table(const TemporaryTables *temporary_tables, const int table_id)
Definition: Execute.h:224
const int8_t * getResultSetColumn(const InputColDescriptor *col_desc, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
int getColId() const
int getTableId() const
#define CHECK(condition)
Definition: Logger.h:197
const InputDescriptor & getScanDesc() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::getResultSetColumn ( const ResultSetPtr buffer,
const int  table_id,
const int  col_id,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const
private

Definition at line 331 of file ColumnFetcher.cpp.

References CHECK_GE, CHECK_NE, columnar_conversion_mutex_, columnarize_result(), columnarized_table_cache_, executor_, run_benchmark_import::result, and transferColumnIfNeeded().

337  {
338  const ColumnarResults* result{nullptr};
339  {
340  std::lock_guard<std::mutex> columnar_conversion_guard(columnar_conversion_mutex_);
341  if (columnarized_table_cache_.empty() || !columnarized_table_cache_.count(table_id)) {
342  columnarized_table_cache_.insert(std::make_pair(
343  table_id, std::unordered_map<int, std::shared_ptr<const ColumnarResults>>()));
344  }
345  auto& frag_id_to_result = columnarized_table_cache_[table_id];
346  int frag_id = 0;
347  if (frag_id_to_result.empty() || !frag_id_to_result.count(frag_id)) {
348  frag_id_to_result.insert(
349  std::make_pair(frag_id,
350  std::shared_ptr<const ColumnarResults>(columnarize_result(
351  executor_->row_set_mem_owner_, buffer, frag_id))));
352  }
353  CHECK_NE(size_t(0), columnarized_table_cache_.count(table_id));
354  result = columnarized_table_cache_[table_id][frag_id].get();
355  }
356  CHECK_GE(col_id, 0);
357  return transferColumnIfNeeded(result,
358  col_id,
359  &executor_->getCatalog()->getDataMgr(),
360  memory_level,
361  device_id,
362  device_allocator);
363 }
#define CHECK_GE(x, y)
Definition: Logger.h:210
ColumnCacheMap columnarized_table_cache_
Executor * executor_
std::mutex columnar_conversion_mutex_
#define CHECK_NE(x, y)
Definition: Logger.h:206
static const int8_t * transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
const ColumnarResults * columnarize_result(std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const ResultSetPtr &result, const int frag_id)
Definition: Execute.h:268

+ Here is the call graph for this function:

JoinColumn ColumnFetcher::makeJoinColumn ( Executor executor,
const Analyzer::ColumnVar hash_col,
const std::vector< Fragmenter_Namespace::FragmentInfo > &  fragments,
const Data_Namespace::MemoryLevel  effective_mem_lvl,
const int  device_id,
DeviceAllocator device_allocator,
std::vector< std::shared_ptr< Chunk_NS::Chunk >> &  chunks_owner,
std::vector< std::shared_ptr< void >> &  malloc_owner,
ColumnCacheMap column_cache 
)
static

Creates a JoinColumn struct containing an array of JoinChunk structs.

makeJoinColumn() creates a JoinColumn struct containing a array of JoinChunk structs, col_chunks_buff, malloced in CPU memory. Although the col_chunks_buff array is in CPU memory here, each JoinChunk struct contains an int8_t* pointer from getOneColumnFragment(), col_buff, that can point to either CPU memory or GPU memory depending on the effective_mem_lvl parameter. See also the fetchJoinColumn() function where col_chunks_buff is copied into GPU memory if needed. The malloc_owner parameter will have the malloced array appended. The chunks_owner parameter will be appended with the chunks.

Definition at line 108 of file ColumnFetcher.cpp.

References CHECK, CHECK_GT, checked_malloc(), JoinChunk::col_buff, Data_Namespace::CPU_LEVEL, SQLTypeInfo::get_size(), Analyzer::Expr::get_type_info(), getOneColumnFragment(), and JoinChunk::num_elems.

Referenced by HashJoin::fetchJoinColumn().

117  {
118  CHECK(!fragments.empty());
119 
120  size_t col_chunks_buff_sz = sizeof(struct JoinChunk) * fragments.size();
121  // TODO: needs an allocator owner
122  auto col_chunks_buff = reinterpret_cast<int8_t*>(
123  malloc_owner.emplace_back(checked_malloc(col_chunks_buff_sz), free).get());
124  auto join_chunk_array = reinterpret_cast<struct JoinChunk*>(col_chunks_buff);
125 
126  size_t num_elems = 0;
127  size_t num_chunks = 0;
128  for (auto& frag : fragments) {
129  auto [col_buff, elem_count] = getOneColumnFragment(
130  executor,
131  hash_col,
132  frag,
133  effective_mem_lvl,
134  effective_mem_lvl == Data_Namespace::CPU_LEVEL ? 0 : device_id,
135  device_allocator,
136  chunks_owner,
137  column_cache);
138  if (col_buff != nullptr) {
139  num_elems += elem_count;
140  join_chunk_array[num_chunks] = JoinChunk{col_buff, elem_count};
141  } else {
142  continue;
143  }
144  ++num_chunks;
145  }
146 
147  int elem_sz = hash_col.get_type_info().get_size();
148  CHECK_GT(elem_sz, 0);
149 
150  return {col_chunks_buff,
151  col_chunks_buff_sz,
152  num_chunks,
153  num_elems,
154  static_cast<size_t>(elem_sz)};
155 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:321
const int8_t * col_buff
#define CHECK_GT(x, y)
Definition: Logger.h:209
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:44
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:78
#define CHECK(condition)
Definition: Logger.h:197
static std::pair< const int8_t *, size_t > getOneColumnFragment(Executor *executor, const Analyzer::ColumnVar &hash_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, ColumnCacheMap &column_cache)
Gets one chunk&#39;s pointer and element count on either CPU or GPU.
size_t num_elems

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::transferColumnIfNeeded ( const ColumnarResults columnar_results,
const int  col_id,
Data_Namespace::DataMgr data_mgr,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
)
staticprivate

Definition at line 308 of file ColumnFetcher.cpp.

References Allocator::alloc(), CHECK, CHECK_LT, DeviceAllocator::copyToDevice(), ColumnarResults::getColumnBuffers(), ColumnarResults::getColumnType(), Data_Namespace::GPU_LEVEL, and ColumnarResults::size().

Referenced by getAllTableColumnFragments(), getOneColumnFragment(), and getResultSetColumn().

314  {
315  if (!columnar_results) {
316  return nullptr;
317  }
318  const auto& col_buffers = columnar_results->getColumnBuffers();
319  CHECK_LT(static_cast<size_t>(col_id), col_buffers.size());
320  if (memory_level == Data_Namespace::GPU_LEVEL) {
321  const auto& col_ti = columnar_results->getColumnType(col_id);
322  const auto num_bytes = columnar_results->size() * col_ti.get_size();
323  CHECK(device_allocator);
324  auto gpu_col_buffer = device_allocator->alloc(num_bytes);
325  device_allocator->copyToDevice(gpu_col_buffer, col_buffers[col_id], num_bytes);
326  return gpu_col_buffer;
327  }
328  return col_buffers[col_id];
329 }
virtual void copyToDevice(int8_t *device_dst, const int8_t *host_src, const size_t num_bytes) const =0
virtual int8_t * alloc(const size_t num_bytes)=0
const size_t size() const
#define CHECK_LT(x, y)
Definition: Logger.h:207
#define CHECK(condition)
Definition: Logger.h:197
const std::vector< int8_t * > & getColumnBuffers() const
const SQLTypeInfo & getColumnType(const int col_id) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Friends And Related Function Documentation

friend class QueryCompilationDescriptor
friend

Definition at line 125 of file ColumnFetcher.h.

friend class TableFunctionExecutionContext
friend

Definition at line 126 of file ColumnFetcher.h.

Member Data Documentation

std::mutex ColumnFetcher::columnar_conversion_mutex_
mutableprivate

Definition at line 116 of file ColumnFetcher.h.

Referenced by getAllTableColumnFragments(), and getResultSetColumn().

std::unordered_map< InputColDescriptor, std::unordered_map<CacheKey, std::unique_ptr<const ColumnarResults> > > ColumnFetcher::columnarized_ref_table_cache_
mutableprivate

Definition at line 121 of file ColumnFetcher.h.

std::unordered_map<InputColDescriptor, std::unique_ptr<const ColumnarResults> > ColumnFetcher::columnarized_scan_table_cache_
mutableprivate

Definition at line 123 of file ColumnFetcher.h.

Referenced by getAllTableColumnFragments().

ColumnCacheMap ColumnFetcher::columnarized_table_cache_
mutableprivate
Executor* ColumnFetcher::executor_
private

The documentation for this class was generated from the following files: