OmniSciDB  95562058bd
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ColumnFetcher Class Reference

#include <ColumnFetcher.h>

+ Collaboration diagram for ColumnFetcher:

Public Member Functions

 ColumnFetcher (Executor *executor, const ColumnCacheMap &column_cache)
 
const int8_t * getOneTableColumnFragment (const int table_id, const int frag_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunk_holder, std::list< ChunkIter > &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 
const int8_t * getAllTableColumnFragments (const int table_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 
const int8_t * getResultSetColumn (const InputColDescriptor *col_desc, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 

Static Public Member Functions

static std::pair< const int8_t
*, size_t > 
getOneColumnFragment (Executor *executor, const Analyzer::ColumnVar &hash_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, ColumnCacheMap &column_cache)
 Gets one chunk's pointer and element count on either CPU or GPU. More...
 
static JoinColumn makeJoinColumn (Executor *executor, const Analyzer::ColumnVar &hash_col, const std::vector< Fragmenter_Namespace::FragmentInfo > &fragments, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, std::vector< std::shared_ptr< void >> &malloc_owner, ColumnCacheMap &column_cache)
 Creates a JoinColumn struct containing an array of JoinChunk structs. More...
 

Private Types

using CacheKey = std::vector< int >
 

Private Member Functions

const int8_t * getResultSetColumn (const ResultSetPtr &buffer, const int table_id, const int col_id, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
 

Static Private Member Functions

static const int8_t * transferColumnIfNeeded (const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
 

Private Attributes

Executorexecutor_
 
std::mutex columnar_conversion_mutex_
 
ColumnCacheMap columnarized_table_cache_
 
std::unordered_map
< InputColDescriptor,
std::unordered_map< CacheKey,
std::unique_ptr< const
ColumnarResults > > > 
columnarized_ref_table_cache_
 
std::unordered_map
< InputColDescriptor,
std::unique_ptr< const
ColumnarResults > > 
columnarized_scan_table_cache_
 

Friends

class QueryCompilationDescriptor
 
class TableFunctionExecutionContext
 

Detailed Description

Definition at line 47 of file ColumnFetcher.h.

Member Typedef Documentation

using ColumnFetcher::CacheKey = std::vector<int>
private

Definition at line 115 of file ColumnFetcher.h.

Constructor & Destructor Documentation

ColumnFetcher::ColumnFetcher ( Executor executor,
const ColumnCacheMap column_cache 
)

Definition at line 23 of file ColumnFetcher.cpp.

24  : executor_(executor), columnarized_table_cache_(column_cache) {}
ColumnCacheMap columnarized_table_cache_
Executor * executor_

Member Function Documentation

const int8_t * ColumnFetcher::getAllTableColumnFragments ( const int  table_id,
const int  col_id,
const std::map< int, const TableFragments * > &  all_tables_fragments,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const

Definition at line 235 of file ColumnFetcher.cpp.

References CHECK, columnar_conversion_mutex_, columnarized_scan_table_cache_, Data_Namespace::CPU_LEVEL, executor_, getOneTableColumnFragment(), InputColDescriptor::getScanDesc(), InputDescriptor::getSourceType(), ColumnarResults::mergeResults(), TABLE, and transferColumnIfNeeded().

Referenced by Executor::fetchChunks(), and Executor::fetchUnionChunks().

241  {
242  const auto fragments_it = all_tables_fragments.find(table_id);
243  CHECK(fragments_it != all_tables_fragments.end());
244  const auto fragments = fragments_it->second;
245  const auto frag_count = fragments->size();
246  std::vector<std::unique_ptr<ColumnarResults>> column_frags;
247  const ColumnarResults* table_column = nullptr;
248  const InputColDescriptor col_desc(col_id, table_id, int(0));
249  CHECK(col_desc.getScanDesc().getSourceType() == InputSourceType::TABLE);
250  {
251  std::lock_guard<std::mutex> columnar_conversion_guard(columnar_conversion_mutex_);
252  auto column_it = columnarized_scan_table_cache_.find(col_desc);
253  if (column_it == columnarized_scan_table_cache_.end()) {
254  for (size_t frag_id = 0; frag_id < frag_count; ++frag_id) {
255  std::list<std::shared_ptr<Chunk_NS::Chunk>> chunk_holder;
256  std::list<ChunkIter> chunk_iter_holder;
257  const auto& fragment = (*fragments)[frag_id];
258  if (fragment.isEmptyPhysicalFragment()) {
259  continue;
260  }
261  auto chunk_meta_it = fragment.getChunkMetadataMap().find(col_id);
262  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
263  auto col_buffer = getOneTableColumnFragment(table_id,
264  static_cast<int>(frag_id),
265  col_id,
266  all_tables_fragments,
267  chunk_holder,
268  chunk_iter_holder,
270  int(0),
271  device_allocator);
272  column_frags.push_back(
273  std::make_unique<ColumnarResults>(executor_->row_set_mem_owner_,
274  col_buffer,
275  fragment.getNumTuples(),
276  chunk_meta_it->second->sqlType));
277  }
278  auto merged_results =
279  ColumnarResults::mergeResults(executor_->row_set_mem_owner_, column_frags);
280  table_column = merged_results.get();
281  columnarized_scan_table_cache_.emplace(col_desc, std::move(merged_results));
282  } else {
283  table_column = column_it->second.get();
284  }
285  }
286  return ColumnFetcher::transferColumnIfNeeded(table_column,
287  0,
288  &executor_->getCatalog()->getDataMgr(),
289  memory_level,
290  device_id,
291  device_allocator);
292 }
static std::unique_ptr< ColumnarResults > mergeResults(const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const std::vector< std::unique_ptr< ColumnarResults >> &sub_results)
Executor * executor_
std::unordered_map< InputColDescriptor, std::unique_ptr< const ColumnarResults > > columnarized_scan_table_cache_
const int8_t * getOneTableColumnFragment(const int table_id, const int frag_id, const int col_id, const std::map< int, const TableFragments * > &all_tables_fragments, std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunk_holder, std::list< ChunkIter > &chunk_iter_holder, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
std::mutex columnar_conversion_mutex_
static const int8_t * transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair< const int8_t *, size_t > ColumnFetcher::getOneColumnFragment ( Executor executor,
const Analyzer::ColumnVar hash_col,
const Fragmenter_Namespace::FragmentInfo fragment,
const Data_Namespace::MemoryLevel  effective_mem_lvl,
const int  device_id,
DeviceAllocator device_allocator,
std::vector< std::shared_ptr< Chunk_NS::Chunk >> &  chunks_owner,
ColumnCacheMap column_cache 
)
static

Gets one chunk's pointer and element count on either CPU or GPU.

Gets a column fragment chunk on CPU or on GPU depending on the effective memory level parameter. For temporary tables, the chunk will be copied to the GPU if needed. Returns a buffer pointer and an element count.

Definition at line 29 of file ColumnFetcher.cpp.

References CHECK, columnarize_result(), Data_Namespace::CPU_LEVEL, Fragmenter_Namespace::FragmentInfo::fragmentId, get_column_descriptor_maybe(), Analyzer::ColumnVar::get_column_id(), Analyzer::ColumnVar::get_table_id(), get_temporary_table(), Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMap(), Fragmenter_Namespace::FragmentInfo::getNumTuples(), Fragmenter_Namespace::FragmentInfo::isEmptyPhysicalFragment(), Fragmenter_Namespace::FragmentInfo::physicalTableId, and transferColumnIfNeeded().

Referenced by RelAlgExecutor::createWindowFunctionContext(), TableFunctionExecutionContext::execute(), and makeJoinColumn().

37  {
38  static std::mutex columnar_conversion_mutex;
39  if (fragment.isEmptyPhysicalFragment()) {
40  return {nullptr, 0};
41  }
42  const auto table_id = hash_col.get_table_id();
43  auto chunk_meta_it = fragment.getChunkMetadataMap().find(hash_col.get_column_id());
44  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
45  const auto& catalog = *executor->getCatalog();
46  const auto cd =
47  get_column_descriptor_maybe(hash_col.get_column_id(), table_id, catalog);
48  CHECK(!cd || !(cd->isVirtualCol));
49  const int8_t* col_buff = nullptr;
50  if (cd) { // real table
51  ChunkKey chunk_key{catalog.getCurrentDB().dbId,
52  fragment.physicalTableId,
53  hash_col.get_column_id(),
54  fragment.fragmentId};
55  const auto chunk = Chunk_NS::Chunk::getChunk(
56  cd,
57  &catalog.getDataMgr(),
58  chunk_key,
59  effective_mem_lvl,
60  effective_mem_lvl == Data_Namespace::CPU_LEVEL ? 0 : device_id,
61  chunk_meta_it->second->numBytes,
62  chunk_meta_it->second->numElements);
63  chunks_owner.push_back(chunk);
64  CHECK(chunk);
65  auto ab = chunk->getBuffer();
66  CHECK(ab->getMemoryPtr());
67  col_buff = reinterpret_cast<int8_t*>(ab->getMemoryPtr());
68  } else { // temporary table
69  const ColumnarResults* col_frag{nullptr};
70  {
71  std::lock_guard<std::mutex> columnar_conversion_guard(columnar_conversion_mutex);
72  const auto frag_id = fragment.fragmentId;
73  if (column_cache.empty() || !column_cache.count(table_id)) {
74  column_cache.insert(std::make_pair(
75  table_id, std::unordered_map<int, std::shared_ptr<const ColumnarResults>>()));
76  }
77  auto& frag_id_to_result = column_cache[table_id];
78  if (frag_id_to_result.empty() || !frag_id_to_result.count(frag_id)) {
79  frag_id_to_result.insert(
80  std::make_pair(frag_id,
81  std::shared_ptr<const ColumnarResults>(columnarize_result(
82  executor->row_set_mem_owner_,
83  get_temporary_table(executor->temporary_tables_, table_id),
84  frag_id))));
85  }
86  col_frag = column_cache[table_id][frag_id].get();
87  }
88  col_buff = transferColumnIfNeeded(
89  col_frag,
90  hash_col.get_column_id(),
91  &catalog.getDataMgr(),
92  effective_mem_lvl,
93  effective_mem_lvl == Data_Namespace::CPU_LEVEL ? 0 : device_id,
94  device_allocator);
95  }
96  return {col_buff, fragment.getNumTuples()};
97 }
int get_table_id() const
Definition: Analyzer.h:194
std::vector< int > ChunkKey
Definition: types.h:37
const ResultSetPtr & get_temporary_table(const TemporaryTables *temporary_tables, const int table_id)
Definition: Execute.h:191
const ColumnDescriptor * get_column_descriptor_maybe(const int col_id, const int table_id, const Catalog_Namespace::Catalog &cat)
Definition: Execute.h:183
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
const ChunkMetadataMap & getChunkMetadataMap() const
static const int8_t * transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
const ColumnarResults * columnarize_result(std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const ResultSetPtr &result, const int frag_id)
Definition: Execute.h:235
#define CHECK(condition)
Definition: Logger.h:197
int get_column_id() const
Definition: Analyzer.h:195

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::getOneTableColumnFragment ( const int  table_id,
const int  frag_id,
const int  col_id,
const std::map< int, const TableFragments * > &  all_tables_fragments,
std::list< std::shared_ptr< Chunk_NS::Chunk >> &  chunk_holder,
std::list< ChunkIter > &  chunk_iter_holder,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const

Definition at line 158 of file ColumnFetcher.cpp.

References Allocator::alloc(), cat(), CHECK, CHECK_EQ, CHECK_GT, DeviceAllocator::copyToDevice(), Data_Namespace::CPU_LEVEL, executor_, get_column_descriptor(), get_column_type(), Chunk_NS::Chunk::getChunk(), Data_Namespace::GPU_LEVEL, and kENCODING_NONE.

Referenced by Executor::fetchChunks(), Executor::fetchUnionChunks(), and getAllTableColumnFragments().

167  {
168  static std::mutex varlen_chunk_mutex; // TODO(alex): remove
169  static std::mutex chunk_list_mutex;
170  const auto fragments_it = all_tables_fragments.find(table_id);
171  CHECK(fragments_it != all_tables_fragments.end());
172  const auto fragments = fragments_it->second;
173  const auto& fragment = (*fragments)[frag_id];
174  if (fragment.isEmptyPhysicalFragment()) {
175  return nullptr;
176  }
177  std::shared_ptr<Chunk_NS::Chunk> chunk;
178  auto chunk_meta_it = fragment.getChunkMetadataMap().find(col_id);
179  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
180  CHECK(table_id > 0);
181  const auto& cat = *executor_->getCatalog();
182  auto cd = get_column_descriptor(col_id, table_id, cat);
183  CHECK(cd);
184  const auto col_type =
185  get_column_type(col_id, table_id, cd, executor_->temporary_tables_);
186  const bool is_real_string =
187  col_type.is_string() && col_type.get_compression() == kENCODING_NONE;
188  const bool is_varlen =
189  is_real_string ||
190  col_type.is_array(); // TODO: should it be col_type.is_varlen_array() ?
191  {
192  ChunkKey chunk_key{
193  cat.getCurrentDB().dbId, fragment.physicalTableId, col_id, fragment.fragmentId};
194  std::unique_ptr<std::lock_guard<std::mutex>> varlen_chunk_lock;
195  if (is_varlen) {
196  varlen_chunk_lock.reset(new std::lock_guard<std::mutex>(varlen_chunk_mutex));
197  }
199  cd,
200  &cat.getDataMgr(),
201  chunk_key,
202  memory_level,
203  memory_level == Data_Namespace::CPU_LEVEL ? 0 : device_id,
204  chunk_meta_it->second->numBytes,
205  chunk_meta_it->second->numElements);
206  std::lock_guard<std::mutex> chunk_list_lock(chunk_list_mutex);
207  chunk_holder.push_back(chunk);
208  }
209  if (is_varlen) {
210  CHECK_GT(table_id, 0);
211  CHECK(chunk_meta_it != fragment.getChunkMetadataMap().end());
212  chunk_iter_holder.push_back(chunk->begin_iterator(chunk_meta_it->second));
213  auto& chunk_iter = chunk_iter_holder.back();
214  if (memory_level == Data_Namespace::CPU_LEVEL) {
215  return reinterpret_cast<int8_t*>(&chunk_iter);
216  } else {
217  auto ab = chunk->getBuffer();
218  ab->pin();
219  auto& row_set_mem_owner = executor_->getRowSetMemoryOwner();
220  row_set_mem_owner->addVarlenInputBuffer(ab);
221  CHECK_EQ(Data_Namespace::GPU_LEVEL, memory_level);
222  CHECK(allocator);
223  auto chunk_iter_gpu = allocator->alloc(sizeof(ChunkIter));
224  allocator->copyToDevice(
225  chunk_iter_gpu, reinterpret_cast<int8_t*>(&chunk_iter), sizeof(ChunkIter));
226  return chunk_iter_gpu;
227  }
228  } else {
229  auto ab = chunk->getBuffer();
230  CHECK(ab->getMemoryPtr());
231  return ab->getMemoryPtr(); // @TODO(alex) change to use ChunkIter
232  }
233 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::vector< int > ChunkKey
Definition: types.h:37
std::string cat(Ts &&...args)
const SQLTypeInfo get_column_type(const int col_id, const int table_id, const ColumnDescriptor *cd, const TemporaryTables *temporary_tables)
Definition: Execute.h:199
Executor * executor_
#define CHECK_GT(x, y)
Definition: Logger.h:209
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
#define CHECK(condition)
Definition: Logger.h:197
const ColumnDescriptor * get_column_descriptor(const int col_id, const int table_id, const Catalog_Namespace::Catalog &cat)
Definition: Execute.h:154

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::getResultSetColumn ( const InputColDescriptor col_desc,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const

Definition at line 294 of file ColumnFetcher.cpp.

References CHECK, executor_, get_temporary_table(), InputColDescriptor::getColId(), InputColDescriptor::getScanDesc(), and InputDescriptor::getTableId().

Referenced by Executor::fetchChunks(), and Executor::fetchUnionChunks().

298  {
299  CHECK(col_desc);
300  const auto table_id = col_desc->getScanDesc().getTableId();
301  return getResultSetColumn(get_temporary_table(executor_->temporary_tables_, table_id),
302  table_id,
303  col_desc->getColId(),
304  memory_level,
305  device_id,
306  device_allocator);
307 }
Executor * executor_
const ResultSetPtr & get_temporary_table(const TemporaryTables *temporary_tables, const int table_id)
Definition: Execute.h:191
const int8_t * getResultSetColumn(const InputColDescriptor *col_desc, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator) const
int getColId() const
int getTableId() const
#define CHECK(condition)
Definition: Logger.h:197
const InputDescriptor & getScanDesc() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::getResultSetColumn ( const ResultSetPtr buffer,
const int  table_id,
const int  col_id,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
) const
private

Definition at line 333 of file ColumnFetcher.cpp.

References CHECK_GE, CHECK_NE, columnar_conversion_mutex_, columnarize_result(), columnarized_table_cache_, executor_, run_benchmark_import::result, and transferColumnIfNeeded().

339  {
340  const ColumnarResults* result{nullptr};
341  {
342  std::lock_guard<std::mutex> columnar_conversion_guard(columnar_conversion_mutex_);
343  if (columnarized_table_cache_.empty() || !columnarized_table_cache_.count(table_id)) {
344  columnarized_table_cache_.insert(std::make_pair(
345  table_id, std::unordered_map<int, std::shared_ptr<const ColumnarResults>>()));
346  }
347  auto& frag_id_to_result = columnarized_table_cache_[table_id];
348  int frag_id = 0;
349  if (frag_id_to_result.empty() || !frag_id_to_result.count(frag_id)) {
350  frag_id_to_result.insert(
351  std::make_pair(frag_id,
352  std::shared_ptr<const ColumnarResults>(columnarize_result(
353  executor_->row_set_mem_owner_, buffer, frag_id))));
354  }
355  CHECK_NE(size_t(0), columnarized_table_cache_.count(table_id));
356  result = columnarized_table_cache_[table_id][frag_id].get();
357  }
358  CHECK_GE(col_id, 0);
359  return transferColumnIfNeeded(result,
360  col_id,
361  &executor_->getCatalog()->getDataMgr(),
362  memory_level,
363  device_id,
364  device_allocator);
365 }
#define CHECK_GE(x, y)
Definition: Logger.h:210
ColumnCacheMap columnarized_table_cache_
Executor * executor_
std::mutex columnar_conversion_mutex_
#define CHECK_NE(x, y)
Definition: Logger.h:206
static const int8_t * transferColumnIfNeeded(const ColumnarResults *columnar_results, const int col_id, Data_Namespace::DataMgr *data_mgr, const Data_Namespace::MemoryLevel memory_level, const int device_id, DeviceAllocator *device_allocator)
const ColumnarResults * columnarize_result(std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const ResultSetPtr &result, const int frag_id)
Definition: Execute.h:235

+ Here is the call graph for this function:

JoinColumn ColumnFetcher::makeJoinColumn ( Executor executor,
const Analyzer::ColumnVar hash_col,
const std::vector< Fragmenter_Namespace::FragmentInfo > &  fragments,
const Data_Namespace::MemoryLevel  effective_mem_lvl,
const int  device_id,
DeviceAllocator device_allocator,
std::vector< std::shared_ptr< Chunk_NS::Chunk >> &  chunks_owner,
std::vector< std::shared_ptr< void >> &  malloc_owner,
ColumnCacheMap column_cache 
)
static

Creates a JoinColumn struct containing an array of JoinChunk structs.

makeJoinColumn() creates a JoinColumn struct containing a array of JoinChunk structs, col_chunks_buff, malloced in CPU memory. Although the col_chunks_buff array is in CPU memory here, each JoinChunk struct contains an int8_t* pointer from getOneColumnFragment(), col_buff, that can point to either CPU memory or GPU memory depending on the effective_mem_lvl parameter. See also the fetchJoinColumn() function where col_chunks_buff is copied into GPU memory if needed. The malloc_owner parameter will have the malloced array appended. The chunks_owner parameter will be appended with the chunks.

Definition at line 108 of file ColumnFetcher.cpp.

References CHECK, CHECK_GT, checked_malloc(), JoinChunk::col_buff, Data_Namespace::CPU_LEVEL, SQLTypeInfo::get_size(), Analyzer::Expr::get_type_info(), getOneColumnFragment(), SQLTypeInfo::is_column(), and JoinChunk::num_elems.

Referenced by JoinHashTableInterface::fetchJoinColumn().

117  {
118  CHECK(!fragments.empty());
119 
120  size_t col_chunks_buff_sz = sizeof(struct JoinChunk) * fragments.size();
121  // TODO: needs an allocator owner
122  auto col_chunks_buff = reinterpret_cast<int8_t*>(
123  malloc_owner.emplace_back(checked_malloc(col_chunks_buff_sz), free).get());
124  auto join_chunk_array = reinterpret_cast<struct JoinChunk*>(col_chunks_buff);
125 
126  size_t num_elems = 0;
127  size_t num_chunks = 0;
128  for (auto& frag : fragments) {
129  auto [col_buff, elem_count] = getOneColumnFragment(
130  executor,
131  hash_col,
132  frag,
133  effective_mem_lvl,
134  effective_mem_lvl == Data_Namespace::CPU_LEVEL ? 0 : device_id,
135  device_allocator,
136  chunks_owner,
137  column_cache);
138  if (col_buff != nullptr) {
139  num_elems += elem_count;
140  join_chunk_array[num_chunks] = JoinChunk{col_buff, elem_count};
141  } else {
142  continue;
143  }
144  ++num_chunks;
145  }
146 
147  CHECK(!hash_col.get_type_info().is_column());
148  int elem_sz = hash_col.get_type_info().get_size();
149  CHECK_GT(elem_sz, 0);
150 
151  return {col_chunks_buff,
152  col_chunks_buff_sz,
153  num_chunks,
154  num_elems,
155  static_cast<size_t>(elem_sz)};
156 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:269
const int8_t * col_buff
#define CHECK_GT(x, y)
Definition: Logger.h:209
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:44
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:78
bool is_column() const
Definition: sqltypes.h:430
#define CHECK(condition)
Definition: Logger.h:197
static std::pair< const int8_t *, size_t > getOneColumnFragment(Executor *executor, const Analyzer::ColumnVar &hash_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, DeviceAllocator *device_allocator, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, ColumnCacheMap &column_cache)
Gets one chunk&#39;s pointer and element count on either CPU or GPU.
size_t num_elems

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * ColumnFetcher::transferColumnIfNeeded ( const ColumnarResults columnar_results,
const int  col_id,
Data_Namespace::DataMgr data_mgr,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_id,
DeviceAllocator device_allocator 
)
staticprivate

Definition at line 309 of file ColumnFetcher.cpp.

References Allocator::alloc(), CHECK, CHECK_LT, DeviceAllocator::copyToDevice(), ColumnarResults::getColumnBuffers(), ColumnarResults::getColumnType(), Data_Namespace::GPU_LEVEL, and ColumnarResults::size().

Referenced by getAllTableColumnFragments(), getOneColumnFragment(), and getResultSetColumn().

315  {
316  if (!columnar_results) {
317  return nullptr;
318  }
319  const auto& col_buffers = columnar_results->getColumnBuffers();
320  CHECK_LT(static_cast<size_t>(col_id), col_buffers.size());
321  if (memory_level == Data_Namespace::GPU_LEVEL) {
322  const auto& col_ti_ = columnar_results->getColumnType(col_id);
323  const auto& col_ti = (col_ti_.is_column() ? col_ti_.get_elem_type() : col_ti_);
324  const auto num_bytes = columnar_results->size() * col_ti.get_size();
325  CHECK(device_allocator);
326  auto gpu_col_buffer = device_allocator->alloc(num_bytes);
327  device_allocator->copyToDevice(gpu_col_buffer, col_buffers[col_id], num_bytes);
328  return gpu_col_buffer;
329  }
330  return col_buffers[col_id];
331 }
virtual void copyToDevice(int8_t *device_dst, const int8_t *host_src, const size_t num_bytes) const =0
virtual int8_t * alloc(const size_t num_bytes)=0
const size_t size() const
#define CHECK_LT(x, y)
Definition: Logger.h:207
#define CHECK(condition)
Definition: Logger.h:197
const std::vector< int8_t * > & getColumnBuffers() const
const SQLTypeInfo & getColumnType(const int col_id) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Friends And Related Function Documentation

friend class QueryCompilationDescriptor
friend

Definition at line 125 of file ColumnFetcher.h.

friend class TableFunctionExecutionContext
friend

Definition at line 126 of file ColumnFetcher.h.

Member Data Documentation

std::mutex ColumnFetcher::columnar_conversion_mutex_
mutableprivate

Definition at line 116 of file ColumnFetcher.h.

Referenced by getAllTableColumnFragments(), and getResultSetColumn().

std::unordered_map< InputColDescriptor, std::unordered_map<CacheKey, std::unique_ptr<const ColumnarResults> > > ColumnFetcher::columnarized_ref_table_cache_
mutableprivate

Definition at line 121 of file ColumnFetcher.h.

std::unordered_map<InputColDescriptor, std::unique_ptr<const ColumnarResults> > ColumnFetcher::columnarized_scan_table_cache_
mutableprivate

Definition at line 123 of file ColumnFetcher.h.

Referenced by getAllTableColumnFragments().

ColumnCacheMap ColumnFetcher::columnarized_table_cache_
mutableprivate
Executor* ColumnFetcher::executor_
private

The documentation for this class was generated from the following files: