OmniSciDB  c07336695a
Chunk.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file Chunk.cpp
19  * @author Wei Hong <wei@mapd.com>
20  */
21 
22 #include "Chunk.h"
23 #include "../DataMgr/ArrayNoneEncoder.h"
24 #include "../DataMgr/FixedLengthArrayNoneEncoder.h"
25 #include "../DataMgr/StringNoneEncoder.h"
26 
27 namespace Chunk_NS {
28 std::shared_ptr<Chunk> Chunk::getChunk(const ColumnDescriptor* cd,
29  DataMgr* data_mgr,
30  const ChunkKey& key,
31  const MemoryLevel memoryLevel,
32  const int deviceId,
33  const size_t numBytes,
34  const size_t numElems) {
35  std::shared_ptr<Chunk> chunkp = std::make_shared<Chunk>(Chunk(cd));
36  chunkp->getChunkBuffer(data_mgr, key, memoryLevel, deviceId, numBytes, numElems);
37  return chunkp;
38 }
39 
41  const ChunkKey& key,
42  const MemoryLevel mem_level,
43  const int device_id) {
45  ChunkKey subKey = key;
46  ChunkKey indexKey(subKey);
47  indexKey.push_back(1);
48  ChunkKey dataKey(subKey);
49  dataKey.push_back(2);
50  return data_mgr->isBufferOnDevice(indexKey, mem_level, device_id) &&
51  data_mgr->isBufferOnDevice(dataKey, mem_level, device_id);
52  } else {
53  return data_mgr->isBufferOnDevice(key, mem_level, device_id);
54  }
55 }
56 
58  const ChunkKey& key,
59  const MemoryLevel mem_level,
60  const int device_id,
61  const size_t num_bytes,
62  const size_t num_elems) {
64  ChunkKey subKey = key;
65  subKey.push_back(1); // 1 for the main buffer
66  buffer = data_mgr->getChunkBuffer(subKey, mem_level, device_id, num_bytes);
67  subKey.pop_back();
68  subKey.push_back(2); // 2 for the index buffer
69  index_buf = data_mgr->getChunkBuffer(
70  subKey,
71  mem_level,
72  device_id,
73  (num_elems + 1) * sizeof(StringOffsetT)); // always record n+1 offsets so string
74  // length can be calculated
75  switch (column_desc->columnType.get_type()) {
76  case kARRAY: {
77  ArrayNoneEncoder* array_encoder =
78  dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
79  array_encoder->set_index_buf(index_buf);
80  break;
81  }
82  case kTEXT:
83  case kVARCHAR:
84  case kCHAR: {
86  StringNoneEncoder* str_encoder =
87  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
88  str_encoder->set_index_buf(index_buf);
89  break;
90  }
91  case kPOINT:
92  case kLINESTRING:
93  case kPOLYGON:
94  case kMULTIPOLYGON: {
95  StringNoneEncoder* str_encoder =
96  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
97  str_encoder->set_index_buf(index_buf);
98  break;
99  }
100  default:
101  CHECK(false);
102  }
103  } else {
104  buffer = data_mgr->getChunkBuffer(key, mem_level, device_id, num_bytes);
105  }
106 }
107 
109  const ChunkKey& key,
110  const MemoryLevel mem_level,
111  const int device_id,
112  const size_t page_size) {
114  ChunkKey subKey = key;
115  subKey.push_back(1); // 1 for the main buffer
116  buffer = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size);
117  subKey.pop_back();
118  subKey.push_back(2); // 2 for the index buffer
119  index_buf = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size);
120  } else {
121  buffer = data_mgr->createChunkBuffer(key, mem_level, device_id, page_size);
122  }
123 }
124 
126  const size_t num_elems,
127  const size_t start_idx,
128  const size_t byte_limit,
129  const bool replicating) {
131  switch (column_desc->columnType.get_type()) {
132  case kARRAY: {
133  if (column_desc->columnType.get_size() > 0) {
134  FixedLengthArrayNoneEncoder* array_encoder =
135  dynamic_cast<FixedLengthArrayNoneEncoder*>(buffer->encoder.get());
136  return array_encoder->getNumElemsForBytesInsertData(
137  src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating);
138  }
139  ArrayNoneEncoder* array_encoder =
140  dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
141  return array_encoder->getNumElemsForBytesInsertData(
142  src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating);
143  }
144  case kTEXT:
145  case kVARCHAR:
146  case kCHAR: {
148  StringNoneEncoder* str_encoder =
149  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
150  return str_encoder->getNumElemsForBytesInsertData(
151  src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating);
152  }
153  case kPOINT:
154  case kLINESTRING:
155  case kPOLYGON:
156  case kMULTIPOLYGON: {
157  StringNoneEncoder* str_encoder =
158  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
159  return str_encoder->getNumElemsForBytesInsertData(
160  src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating);
161  }
162  default:
163  CHECK(false);
164  return 0;
165  }
166 }
167 
169  const size_t num_elems,
170  const size_t start_idx,
171  const bool replicating) {
172  const auto& ti = column_desc->columnType;
173  if (ti.is_varlen()) {
174  switch (ti.get_type()) {
175  case kARRAY: {
176  if (ti.get_size() > 0) {
177  FixedLengthArrayNoneEncoder* array_encoder =
178  dynamic_cast<FixedLengthArrayNoneEncoder*>(buffer->encoder.get());
179  return array_encoder->appendData(
180  src_data.arraysPtr, start_idx, num_elems, replicating);
181  }
182  ArrayNoneEncoder* array_encoder =
183  dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
184  return array_encoder->appendData(
185  src_data.arraysPtr, start_idx, num_elems, replicating);
186  }
187  case kTEXT:
188  case kVARCHAR:
189  case kCHAR: {
190  CHECK_EQ(kENCODING_NONE, ti.get_compression());
191  StringNoneEncoder* str_encoder =
192  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
193  return str_encoder->appendData(
194  src_data.stringsPtr, start_idx, num_elems, replicating);
195  }
196  case kPOINT:
197  case kLINESTRING:
198  case kPOLYGON:
199  case kMULTIPOLYGON: {
200  StringNoneEncoder* str_encoder =
201  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
202  return str_encoder->appendData(
203  src_data.stringsPtr, start_idx, num_elems, replicating);
204  }
205  default:
206  CHECK(false);
207  }
208  }
209  return buffer->encoder->appendData(src_data.numbersPtr, num_elems, ti, replicating);
210 }
211 
213  if (buffer != nullptr) {
214  buffer->unPin();
215  }
216  if (index_buf != nullptr) {
217  index_buf->unPin();
218  }
219 }
220 
224  switch (column_desc->columnType.get_type()) {
225  case kARRAY: {
226  ArrayNoneEncoder* array_encoder =
227  dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
228  array_encoder->set_index_buf(index_buf);
229  break;
230  }
231  case kTEXT:
232  case kVARCHAR:
233  case kCHAR: {
235  StringNoneEncoder* str_encoder =
236  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
237  str_encoder->set_index_buf(index_buf);
238  break;
239  }
240  case kPOINT:
241  case kLINESTRING:
242  case kPOLYGON:
243  case kMULTIPOLYGON: {
244  StringNoneEncoder* str_encoder =
245  dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
246  str_encoder->set_index_buf(index_buf);
247  break;
248  }
249  default:
250  CHECK(false);
251  }
252  }
253 }
254 
256  int start_idx,
257  int skip) const {
258  ChunkIter it;
260  it.skip = skip;
262  if (it.skip_size < 0) { // if it's variable length
263  it.current_pos = it.start_pos =
264  index_buf->getMemoryPtr() + start_idx * sizeof(StringOffsetT);
265  it.end_pos = index_buf->getMemoryPtr() + index_buf->size() - sizeof(StringOffsetT);
267  } else {
268  it.current_pos = it.start_pos = buffer->getMemoryPtr() + start_idx * it.skip_size;
269  it.end_pos = buffer->getMemoryPtr() + buffer->size();
270  it.second_buf = nullptr;
271  }
272  it.num_elems = chunk_metadata.numElements;
273  return it;
274 }
275 } // namespace Chunk_NS
#define CHECK_EQ(x, y)
Definition: Logger.h:195
size_t getNumElemsForBytesInsertData(const std::vector< ArrayDatum > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)
int8_t * start_pos
Definition: ChunkIter.h:33
HOST DEVICE int get_size() const
Definition: sqltypes.h:329
int8_t * current_pos
Definition: ChunkIter.h:32
SQLTypeInfo type_info
Definition: ChunkIter.h:30
std::vector< std::string > * stringsPtr
Definition: sqltypes.h:138
std::vector< ArrayDatum > * arraysPtr
Definition: sqltypes.h:139
void initEncoder(const SQLTypeInfo tmpSqlType)
virtual size_t size() const =0
virtual int8_t * getMemoryPtr()=0
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:319
bool is_varlen() const
Definition: sqltypes.h:460
const ColumnDescriptor * column_desc
Definition: Chunk.h:104
void getChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t num_bytes=0, const size_t num_elems=0)
Definition: Chunk.cpp:57
void init_encoder()
Definition: Chunk.cpp:221
AbstractBuffer * index_buf
Definition: Chunk.h:103
void set_index_buf(AbstractBuffer *buf)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:327
void unpin_buffer()
Definition: Chunk.cpp:212
void set_index_buf(AbstractBuffer *buf)
size_t getNumElemsForBytesInsertData(const std::vector< std::string > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)
static unsigned skip
Definition: testing.h:31
int32_t StringOffsetT
Definition: sqltypes.h:873
int8_t * end_pos
Definition: ChunkIter.h:34
size_t num_elems
Definition: ChunkIter.h:37
ChunkMetadata appendData(DataBlockPtr &srcData, const size_t numAppendElems, const size_t startIdx, const bool replicating=false)
Definition: Chunk.cpp:168
bool isChunkOnDevice(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int device_id)
Definition: Chunk.cpp:40
AbstractBuffer * buffer
Definition: Chunk.h:102
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
specifies the content in-memory of a row in the column metadata table
ChunkMetadata appendData(int8_t *&srcData, const size_t numAppendElems, const SQLTypeInfo &, const bool replicating=false) override
int skip_size
Definition: ChunkIter.h:36
Definition: sqltypes.h:54
bool isBufferOnDevice(const ChunkKey &key, const MemoryLevel memLevel, const int deviceId)
Definition: DataMgr.cpp:317
AbstractBuffer * getChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0)
Definition: DataMgr.cpp:344
ChunkMetadata appendData(int8_t *&srcData, const size_t numAppendElems, const SQLTypeInfo &, const bool replicating=false) override
int8_t * second_buf
Definition: ChunkIter.h:31
Definition: sqltypes.h:43
ChunkIter begin_iterator(const ChunkMetadata &, int start_idx=0, int skip=1) const
Definition: Chunk.cpp:255
int skip
Definition: ChunkIter.h:35
#define CHECK(condition)
Definition: Logger.h:187
std::vector< int > ChunkKey
Definition: types.h:35
AbstractBuffer * createChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0)
Definition: DataMgr.cpp:336
size_t getNumElemsForBytesInsertData(const DataBlockPtr &src_data, const size_t num_elems, const size_t start_idx, const size_t byte_limit, const bool replicating=false)
Definition: Chunk.cpp:125
void createChunkBuffer(DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t page_size=0)
Definition: Chunk.cpp:108
SQLTypeInfo columnType
int8_t * numbersPtr
Definition: sqltypes.h:137
size_t getNumElemsForBytesInsertData(const std::vector< ArrayDatum > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)
std::unique_ptr< Encoder > encoder
ChunkMetadata appendData(int8_t *&srcData, const size_t numAppendElems, const SQLTypeInfo &, const bool replicating=false) override
size_t numElements
Definition: ChunkMetadata.h:34
bool is_fixlen_array() const
Definition: sqltypes.h:456