OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Chunk.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #pragma once
24 
25 #include <list>
26 #include <memory>
27 
28 #include "Catalog/CatalogFwd.h"
29 #include "DataMgr/AbstractBuffer.h"
30 #include "DataMgr/ChunkMetadata.h"
31 #include "DataMgr/DataMgr.h"
32 #include "Shared/sqltypes.h"
33 #include "Utils/ChunkIter.h"
34 
38 
39 namespace Chunk_NS {
40 
41 class Chunk {
42  public:
43  Chunk(bool pinnable = true)
44  : buffer_(nullptr)
45  , index_buf_(nullptr)
46  , column_desc_(nullptr)
47  , pinnable_(pinnable) {}
48 
49  explicit Chunk(const ColumnDescriptor* td)
50  : buffer_(nullptr), index_buf_(nullptr), column_desc_(td), pinnable_(true) {}
51 
52  Chunk(const ColumnDescriptor* td, bool pinnable)
53  : buffer_(nullptr), index_buf_(nullptr), column_desc_(td), pinnable_(pinnable) {}
54 
56  AbstractBuffer* ib,
57  const ColumnDescriptor* td,
58  bool pinnable = true)
59  : buffer_(b), index_buf_(ib), column_desc_(td), pinnable_(pinnable) {}
60 
62 
63  void setPinnable(bool pinnable) { pinnable_ = pinnable; }
64 
65  const ColumnDescriptor* getColumnDesc() const { return column_desc_; }
66 
67  void setColumnDesc(const ColumnDescriptor* cd) { column_desc_ = cd; }
68 
70  const std::list<const ColumnDescriptor*>& colDescs,
71  std::vector<Chunk>& chunkVec);
72 
73  ChunkIter begin_iterator(const std::shared_ptr<ChunkMetadata>&,
74  int start_idx = 0,
75  int skip = 1) const;
76 
77  size_t getNumElemsForBytesEncodedDataAtIndices(const int8_t* index_data,
78  const std::vector<size_t>& selected_idx,
79  const size_t byte_limit);
80 
81  size_t getNumElemsForBytesInsertData(const DataBlockPtr& src_data,
82  const size_t num_elems,
83  const size_t start_idx,
84  const size_t byte_limit,
85  const bool replicating = false);
86 
87  std::shared_ptr<ChunkMetadata> appendData(DataBlockPtr& srcData,
88  const size_t numAppendElems,
89  const size_t startIdx,
90  const bool replicating = false);
91 
92  std::shared_ptr<ChunkMetadata> appendEncodedDataAtIndices(
93  const Chunk& src_chunk,
94  const std::vector<size_t>& selected_idx);
95 
96  std::shared_ptr<ChunkMetadata> appendEncodedData(const Chunk& src_chunk,
97  const size_t num_elements,
98  const size_t start_idx);
99 
100  void createChunkBuffer(DataMgr* data_mgr,
101  const ChunkKey& key,
102  const MemoryLevel mem_level,
103  const int deviceId = 0,
104  const size_t page_size = 0);
105 
106  void getChunkBuffer(DataMgr* data_mgr,
107  const ChunkKey& key,
108  const MemoryLevel mem_level,
109  const int deviceId = 0,
110  const size_t num_bytes = 0,
111  const size_t num_elems = 0);
112 
113  static std::shared_ptr<Chunk> getChunk(const ColumnDescriptor* cd,
114  DataMgr* data_mgr,
115  const ChunkKey& key,
116  const MemoryLevel mem_level,
117  const int deviceId,
118  const size_t num_bytes,
119  const size_t num_elems,
120  const bool pinnable = true);
121 
136  static std::shared_ptr<Chunk> getChunk(const ColumnDescriptor* cd,
137  AbstractBuffer* data_buffer,
138  AbstractBuffer* index_buffer,
139  const bool pinnable = true);
140 
141  bool isChunkOnDevice(DataMgr* data_mgr,
142  const ChunkKey& key,
143  const MemoryLevel mem_level,
144  const int device_id);
145 
146  AbstractBuffer* getBuffer() const { return buffer_; }
147 
148  AbstractBuffer* getIndexBuf() const { return index_buf_; }
149 
150  void setBuffer(AbstractBuffer* b) { buffer_ = b; }
151 
153 
154  void initEncoder();
155 
156  void decompress(int8_t* compressed, VarlenDatum* result, Datum* datum) const;
157 
158  std::string toString() const;
159 
160  private:
161  void setChunkBuffer(AbstractBuffer* buffer, AbstractBuffer* index_buffer);
162 
166  // When using Chunk as a buffer wrapper, disable pinnable_ to avoid assymetric pin/unPin
167  // of the buffers
168  bool pinnable_;
169 
170  void unpinBuffer();
171 };
172 
173 } // namespace Chunk_NS
Chunk(const ColumnDescriptor *td, bool pinnable)
Definition: Chunk.h:52
AbstractBuffer * getIndexBuf() const
Definition: Chunk.h:148
void setPinnable(bool pinnable)
Definition: Chunk.h:63
Chunk(const ColumnDescriptor *td)
Definition: Chunk.h:49
std::vector< int > ChunkKey
Definition: types.h:36
const ColumnDescriptor * column_desc_
Definition: Chunk.h:165
void setIndexBuffer(AbstractBuffer *ib)
Definition: Chunk.h:152
void getChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t num_bytes=0, const size_t num_elems=0)
Definition: Chunk.cpp:112
std::shared_ptr< ChunkMetadata > appendEncodedDataAtIndices(const Chunk &src_chunk, const std::vector< size_t > &selected_idx)
Definition: Chunk.cpp:208
void setChunkBuffer(AbstractBuffer *buffer, AbstractBuffer *index_buffer)
Definition: Chunk.cpp:71
Constants for Builtin SQL Types supported by HEAVY.AI.
void setBuffer(AbstractBuffer *b)
Definition: Chunk.h:150
const ColumnDescriptor * getColumnDesc() const
Definition: Chunk.h:65
AbstractBuffer * buffer_
Definition: Chunk.h:163
void unpinBuffer()
Definition: Chunk.cpp:279
bool isChunkOnDevice(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int device_id)
Definition: Chunk.cpp:53
An AbstractBuffer is a unit of data management for a data manager.
specifies the content in-memory of a row in the column metadata table
bool g_enable_smem_group_by true
Chunk(AbstractBuffer *b, AbstractBuffer *ib, const ColumnDescriptor *td, bool pinnable=true)
Definition: Chunk.h:55
AbstractBuffer * getBuffer() const
Definition: Chunk.h:146
void initEncoder()
Definition: Chunk.cpp:290
size_t getNumElemsForBytesEncodedDataAtIndices(const int8_t *index_data, const std::vector< size_t > &selected_idx, const size_t byte_limit)
Definition: Chunk.cpp:153
std::shared_ptr< ChunkMetadata > appendEncodedData(const Chunk &src_chunk, const size_t num_elements, const size_t start_idx)
Definition: Chunk.cpp:220
void setColumnDesc(const ColumnDescriptor *cd)
Definition: Chunk.h:67
std::string toString() const
Definition: Chunk.cpp:356
size_t getNumElemsForBytesInsertData(const DataBlockPtr &src_data, const size_t num_elems, const size_t start_idx, const size_t byte_limit, const bool replicating=false)
Definition: Chunk.cpp:163
ChunkIter begin_iterator(const std::shared_ptr< ChunkMetadata > &, int start_idx=0, int skip=1) const
Definition: Chunk.cpp:327
void createChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t page_size=0)
Definition: Chunk.cpp:135
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems, const bool pinnable=true)
Definition: Chunk.cpp:31
void decompress(int8_t *compressed, VarlenDatum *result, Datum *datum) const
AbstractBuffer * index_buf_
Definition: Chunk.h:164
Definition: Datum.h:69
Chunk(bool pinnable=true)
Definition: Chunk.h:43
bool pinnable_
Definition: Chunk.h:168
std::shared_ptr< ChunkMetadata > appendData(DataBlockPtr &srcData, const size_t numAppendElems, const size_t startIdx, const bool replicating=false)
Definition: Chunk.cpp:232
static void translateColumnDescriptorsToChunkVec(const std::list< const ColumnDescriptor * > &colDescs, std::vector< Chunk > &chunkVec)
Definition: Chunk.cpp:348