OmniSciDB  91042dcc5b
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ArrayNoneEncoder.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
24 #ifndef ARRAY_NONE_ENCODER_H
25 #define ARRAY_NONE_ENCODER_H
26 
27 #include "Logger/Logger.h"
28 
29 #include <cassert>
30 #include <cstring>
31 #include <memory>
32 #include <mutex>
33 #include <string>
34 #include <vector>
35 #include "AbstractBuffer.h"
36 #include "ChunkMetadata.h"
37 #include "Encoder.h"
38 
40 
41 class ArrayNoneEncoder : public Encoder {
42  public:
44  : Encoder(buffer)
45  , has_nulls(false)
47  , index_buf(nullptr)
48  , last_offset(-1) {}
49 
50  size_t getNumElemsForBytesInsertData(const std::vector<ArrayDatum>* srcData,
51  const int start_idx,
52  const size_t numAppendElems,
53  const size_t byteLimit,
54  const bool replicating = false) {
55  size_t dataSize = 0;
56 
57  size_t n = start_idx;
58  for (; n < start_idx + numAppendElems; n++) {
59  size_t len = (*srcData)[replicating ? 0 : n].length;
60  if (dataSize + len > byteLimit) {
61  break;
62  }
63  dataSize += len;
64  }
65  return n - start_idx;
66  }
67 
68  size_t getNumElemsForBytesEncodedData(const int8_t* index_data,
69  const int start_idx,
70  const size_t num_elements,
71  const size_t byte_limit) override {
72  size_t data_size = 0;
73  auto array_offsets = reinterpret_cast<const ArrayOffsetT*>(index_data);
74  size_t count;
75  for (count = 1; count <= num_elements; ++count) {
76  auto current_index = start_idx + count;
77  auto offset = array_offsets[current_index];
78  int64_t last_offset = array_offsets[current_index - 1];
79  size_t array_byte_size = std::abs(offset) - std::abs(last_offset);
80  if (data_size + array_byte_size > byte_limit) {
81  break;
82  }
83  data_size += array_byte_size;
84  }
85  return count - 1;
86  }
87 
88  std::shared_ptr<ChunkMetadata> appendData(int8_t*& src_data,
89  const size_t num_elems_to_append,
90  const SQLTypeInfo& ti,
91  const bool replicating = false,
92  const int64_t offset = -1) override {
93  UNREACHABLE(); // should never be called for arrays
94  return nullptr;
95  }
96 
97  std::shared_ptr<ChunkMetadata> appendEncodedDataAtIndices(
98  const int8_t* index_data,
99  int8_t* data,
100  const std::vector<size_t>& selected_idx) override {
101  std::vector<ArrayDatum> data_subset;
102  data_subset.reserve(selected_idx.size());
103  for (const auto& array_index : selected_idx) {
104  data_subset.emplace_back(getArrayDatumAtIndex(index_data, data, array_index));
105  }
106  return appendData(&data_subset, 0, selected_idx.size(), false);
107  }
108 
109  std::shared_ptr<ChunkMetadata> appendEncodedData(const int8_t* index_data,
110  int8_t* data,
111  const size_t start_idx,
112  const size_t num_elements) override {
113  std::vector<ArrayDatum> data_subset;
114  data_subset.reserve(num_elements);
115  for (size_t count = 0; count < num_elements; ++count) {
116  auto current_index = start_idx + count;
117  data_subset.emplace_back(getArrayDatumAtIndex(index_data, data, current_index));
118  }
119  return appendData(&data_subset, 0, num_elements, false);
120  }
121 
122  std::shared_ptr<ChunkMetadata> appendData(const std::vector<ArrayDatum>* srcData,
123  const int start_idx,
124  const size_t numAppendElems,
125  const bool replicating) {
126  CHECK(index_buf != nullptr); // index_buf must be set before this.
127  size_t index_size = numAppendElems * sizeof(ArrayOffsetT);
128  if (num_elems_ == 0) {
129  index_size += sizeof(ArrayOffsetT); // plus one for the initial offset
130  }
131  index_buf->reserve(index_size);
132 
133  bool first_elem_padded = false;
134  ArrayOffsetT initial_offset = 0;
135  if (num_elems_ == 0) {
136  if ((*srcData)[0].is_null || (*srcData)[0].length <= 1) {
137  // Covers following potentially problematic first arrays:
138  // (1) NULL array, issue - can't encode a NULL with 0 initial offset
139  // otherwise, if first array is not NULL:
140  // (2) length=1 array - could be followed by a {}*/NULL, covers tinyint,bool
141  // (3) empty array - could be followed by {}*/NULL, or {}*|{x}|{}*|NULL, etc.
142  initial_offset = DEFAULT_NULL_PADDING_SIZE;
143  first_elem_padded = true;
144  }
145  index_buf->append((int8_t*)&initial_offset,
146  sizeof(ArrayOffsetT)); // write the initial offset
147  last_offset = initial_offset;
148  } else {
149  // Valid last_offset is never negative
150  // always need to read a valid last offset from buffer/disk
151  // b/c now due to vacuum "last offset" may go backward and if
152  // index chunk was not reloaded last_offset would go way off!
153  index_buf->read((int8_t*)&last_offset,
154  sizeof(ArrayOffsetT),
155  index_buf->size() - sizeof(ArrayOffsetT),
157  CHECK(last_offset != -1);
158  // If the loaded offset is negative it means the last value was a NULL array,
159  // convert to a valid last offset
160  if (last_offset < 0) {
162  }
163  }
164  // Need to start data from 8 byte offset if first array encoded is a NULL array
165  size_t data_size = (first_elem_padded) ? DEFAULT_NULL_PADDING_SIZE : 0;
166  for (size_t n = start_idx; n < start_idx + numAppendElems; n++) {
167  // NULL arrays don't take any space so don't add to the data size
168  if ((*srcData)[replicating ? 0 : n].is_null) {
169  continue;
170  }
171  data_size += (*srcData)[replicating ? 0 : n].length;
172  }
173  buffer_->reserve(data_size);
174 
175  size_t inbuf_size =
176  std::min(std::max(index_size, data_size), (size_t)MAX_INPUT_BUF_SIZE);
177  auto gc_inbuf = std::make_unique<int8_t[]>(inbuf_size);
178  auto inbuf = gc_inbuf.get();
179  for (size_t num_appended = 0; num_appended < numAppendElems;) {
180  ArrayOffsetT* p = (ArrayOffsetT*)inbuf;
181  size_t i;
182  for (i = 0; num_appended < numAppendElems && i < inbuf_size / sizeof(ArrayOffsetT);
183  i++, num_appended++) {
184  p[i] =
185  last_offset + (*srcData)[replicating ? 0 : num_appended + start_idx].length;
186  last_offset = p[i];
187  if ((*srcData)[replicating ? 0 : num_appended + start_idx].is_null) {
188  // Record array NULLness in the index buffer
189  p[i] = -p[i];
190  }
191  }
192  index_buf->append(inbuf, i * sizeof(ArrayOffsetT));
193  }
194 
195  // Pad buffer_ with 8 bytes if first encoded array is a NULL array
196  if (first_elem_padded) {
197  auto padding_size = DEFAULT_NULL_PADDING_SIZE;
198  buffer_->append(inbuf, padding_size);
199  }
200  for (size_t num_appended = 0; num_appended < numAppendElems;) {
201  size_t size = 0;
202  for (int i = start_idx + num_appended;
203  num_appended < numAppendElems && size < inbuf_size;
204  i++, num_appended++) {
205  if ((*srcData)[replicating ? 0 : i].is_null) {
206  continue; // NULL arrays don't take up any space in the data buffer
207  }
208  size_t len = (*srcData)[replicating ? 0 : i].length;
209  if (len > inbuf_size) {
210  // for large strings, append on its own
211  if (size > 0) {
212  buffer_->append(inbuf, size);
213  }
214  size = 0;
215  buffer_->append((*srcData)[replicating ? 0 : i].pointer, len);
216  num_appended++;
217  break;
218  } else if (size + len > inbuf_size) {
219  break;
220  }
221  char* dest = (char*)inbuf + size;
222  if (len > 0) {
223  std::memcpy((void*)dest, (void*)(*srcData)[replicating ? 0 : i].pointer, len);
224  size += len;
225  }
226  }
227  if (size > 0) {
228  buffer_->append(inbuf, size);
229  }
230  }
231  // make sure buffer_ is flushed even if no new data is appended to it
232  // (e.g. empty strings) because the metadata needs to be flushed.
233  if (!buffer_->isDirty()) {
234  buffer_->setDirty();
235  }
236 
237  // keep Chunk statistics with array elements
238  for (size_t n = start_idx; n < start_idx + numAppendElems; n++) {
239  update_elem_stats((*srcData)[replicating ? 0 : n]);
240  }
241  num_elems_ += numAppendElems;
242  auto chunk_metadata = std::make_shared<ChunkMetadata>();
243  getMetadata(chunk_metadata);
244  return chunk_metadata;
245  }
246 
247  void getMetadata(const std::shared_ptr<ChunkMetadata>& chunkMetadata) override {
248  Encoder::getMetadata(chunkMetadata); // call on parent class
249  chunkMetadata->fillChunkStats(elem_min, elem_max, has_nulls);
250  }
251 
252  // Only called from the executor for synthesized meta-information.
253  std::shared_ptr<ChunkMetadata> getMetadata(const SQLTypeInfo& ti) override {
254  auto chunk_metadata = std::make_shared<ChunkMetadata>(
255  ti, 0, 0, ChunkStats{elem_min, elem_max, has_nulls});
256  return chunk_metadata;
257  }
258 
259  void updateStats(const int64_t, const bool) override { CHECK(false); }
260 
261  void updateStats(const double, const bool) override { CHECK(false); }
262 
263  void updateStats(const int8_t* const src_data, const size_t num_elements) override {
264  CHECK(false);
265  }
266 
267  void updateStats(const std::vector<std::string>* const src_data,
268  const size_t start_idx,
269  const size_t num_elements) override {
270  UNREACHABLE();
271  }
272 
273  void updateStats(const std::vector<ArrayDatum>* const src_data,
274  const size_t start_idx,
275  const size_t num_elements) override {
276  for (size_t n = start_idx; n < start_idx + num_elements; n++) {
277  update_elem_stats((*src_data)[n]);
278  }
279  }
280 
281  void reduceStats(const Encoder&) override { CHECK(false); }
282 
283  void writeMetadata(FILE* f) override {
284  // assumes pointer is already in right place
285  fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
286  fwrite((int8_t*)&elem_min, sizeof(Datum), 1, f);
287  fwrite((int8_t*)&elem_max, sizeof(Datum), 1, f);
288  fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
289  fwrite((int8_t*)&initialized, sizeof(bool), 1, f);
290  }
291 
292  void readMetadata(FILE* f) override {
293  // assumes pointer is already in right place
294  fread((int8_t*)&num_elems_, sizeof(size_t), 1, f);
295  fread((int8_t*)&elem_min, sizeof(Datum), 1, f);
296  fread((int8_t*)&elem_max, sizeof(Datum), 1, f);
297  fread((int8_t*)&has_nulls, sizeof(bool), 1, f);
298  fread((int8_t*)&initialized, sizeof(bool), 1, f);
299  }
300 
301  void copyMetadata(const Encoder* copyFromEncoder) override {
302  num_elems_ = copyFromEncoder->getNumElems();
303  auto array_encoder = dynamic_cast<const ArrayNoneEncoder*>(copyFromEncoder);
304  elem_min = array_encoder->elem_min;
305  elem_max = array_encoder->elem_max;
306  has_nulls = array_encoder->has_nulls;
307  initialized = array_encoder->initialized;
308  }
309 
310  AbstractBuffer* getIndexBuf() const { return index_buf; }
311 
312  bool resetChunkStats(const ChunkStats& stats) override {
313  auto elem_type = buffer_->getSqlType().get_elem_type();
314  if (initialized && DatumEqual(elem_min, stats.min, elem_type) &&
315  DatumEqual(elem_max, stats.max, elem_type) && has_nulls == stats.has_nulls) {
316  return false;
317  }
318  elem_min = stats.min;
319  elem_max = stats.max;
320  has_nulls = stats.has_nulls;
321  return true;
322  }
323 
324  void resetChunkStats() override {
325  has_nulls = false;
326  initialized = false;
327  }
328 
331  bool has_nulls;
334  std::unique_lock<std::mutex> lock(EncoderMutex_);
335  index_buf = buf;
336  }
337 
338  static constexpr size_t DEFAULT_NULL_PADDING_SIZE{8};
339 
340  private:
341  std::mutex EncoderMutex_;
344 
345  void update_elem_stats(const ArrayDatum& array) {
346  if (array.is_null) {
347  has_nulls = true;
348  }
349  switch (buffer_->getSqlType().get_subtype()) {
350  case kBOOLEAN: {
351  if (!initialized) {
352  elem_min.boolval = 1;
353  elem_max.boolval = 0;
354  }
355  if (array.is_null || array.length == 0) {
356  break;
357  }
358  const int8_t* bool_array = array.pointer;
359  for (size_t i = 0; i < array.length / sizeof(bool); i++) {
360  if (bool_array[i] == NULL_BOOLEAN) {
361  has_nulls = true;
362  } else if (initialized) {
363  elem_min.boolval = std::min(elem_min.boolval, bool_array[i]);
364  elem_max.boolval = std::max(elem_max.boolval, bool_array[i]);
365  } else {
366  elem_min.boolval = bool_array[i];
367  elem_max.boolval = bool_array[i];
368  initialized = true;
369  }
370  }
371  break;
372  }
373  case kINT: {
374  if (!initialized) {
375  elem_min.intval = 1;
376  elem_max.intval = 0;
377  }
378  if (array.is_null || array.length == 0) {
379  break;
380  }
381  const int32_t* int_array = (int32_t*)array.pointer;
382  for (size_t i = 0; i < array.length / sizeof(int32_t); i++) {
383  if (int_array[i] == NULL_INT) {
384  has_nulls = true;
385  } else if (initialized) {
386  elem_min.intval = std::min(elem_min.intval, int_array[i]);
387  elem_max.intval = std::max(elem_max.intval, int_array[i]);
388  } else {
389  elem_min.intval = int_array[i];
390  elem_max.intval = int_array[i];
391  initialized = true;
392  }
393  }
394  break;
395  }
396  case kSMALLINT: {
397  if (!initialized) {
398  elem_min.smallintval = 1;
399  elem_max.smallintval = 0;
400  }
401  if (array.is_null || array.length == 0) {
402  break;
403  }
404  const int16_t* int_array = (int16_t*)array.pointer;
405  for (size_t i = 0; i < array.length / sizeof(int16_t); i++) {
406  if (int_array[i] == NULL_SMALLINT) {
407  has_nulls = true;
408  } else if (initialized) {
409  elem_min.smallintval = std::min(elem_min.smallintval, int_array[i]);
410  elem_max.smallintval = std::max(elem_max.smallintval, int_array[i]);
411  } else {
412  elem_min.smallintval = int_array[i];
413  elem_max.smallintval = int_array[i];
414  initialized = true;
415  }
416  }
417  break;
418  }
419  case kTINYINT: {
420  if (!initialized) {
421  elem_min.tinyintval = 1;
422  elem_max.tinyintval = 0;
423  }
424  if (array.is_null || array.length == 0) {
425  break;
426  }
427  const int8_t* int_array = (int8_t*)array.pointer;
428  for (size_t i = 0; i < array.length / sizeof(int8_t); i++) {
429  if (int_array[i] == NULL_TINYINT) {
430  has_nulls = true;
431  } else if (initialized) {
432  elem_min.tinyintval = std::min(elem_min.tinyintval, int_array[i]);
433  elem_max.tinyintval = std::max(elem_max.tinyintval, int_array[i]);
434  } else {
435  elem_min.tinyintval = int_array[i];
436  elem_max.tinyintval = int_array[i];
437  initialized = true;
438  }
439  }
440  break;
441  }
442  case kBIGINT:
443  case kNUMERIC:
444  case kDECIMAL: {
445  if (!initialized) {
446  elem_min.bigintval = 1;
447  elem_max.bigintval = 0;
448  }
449  if (array.is_null || array.length == 0) {
450  break;
451  }
452  const int64_t* int_array = (int64_t*)array.pointer;
453  for (size_t i = 0; i < array.length / sizeof(int64_t); i++) {
454  if (int_array[i] == NULL_BIGINT) {
455  has_nulls = true;
456  } else if (initialized) {
457  elem_min.bigintval = std::min(elem_min.bigintval, int_array[i]);
458  elem_max.bigintval = std::max(elem_max.bigintval, int_array[i]);
459  } else {
460  elem_min.bigintval = int_array[i];
461  elem_max.bigintval = int_array[i];
462  initialized = true;
463  }
464  }
465  break;
466  }
467  case kFLOAT: {
468  if (!initialized) {
469  elem_min.floatval = 1.0;
470  elem_max.floatval = 0.0;
471  }
472  if (array.is_null || array.length == 0) {
473  break;
474  }
475  const float* flt_array = (float*)array.pointer;
476  for (size_t i = 0; i < array.length / sizeof(float); i++) {
477  if (flt_array[i] == NULL_FLOAT) {
478  has_nulls = true;
479  } else if (initialized) {
480  elem_min.floatval = std::min(elem_min.floatval, flt_array[i]);
481  elem_max.floatval = std::max(elem_max.floatval, flt_array[i]);
482  } else {
483  elem_min.floatval = flt_array[i];
484  elem_max.floatval = flt_array[i];
485  initialized = true;
486  }
487  }
488  break;
489  }
490  case kDOUBLE: {
491  if (!initialized) {
492  elem_min.doubleval = 1.0;
493  elem_max.doubleval = 0.0;
494  }
495  if (array.is_null || array.length == 0) {
496  break;
497  }
498  const double* dbl_array = (double*)array.pointer;
499  for (size_t i = 0; i < array.length / sizeof(double); i++) {
500  if (dbl_array[i] == NULL_DOUBLE) {
501  has_nulls = true;
502  } else if (initialized) {
503  elem_min.doubleval = std::min(elem_min.doubleval, dbl_array[i]);
504  elem_max.doubleval = std::max(elem_max.doubleval, dbl_array[i]);
505  } else {
506  elem_min.doubleval = dbl_array[i];
507  elem_max.doubleval = dbl_array[i];
508  initialized = true;
509  }
510  }
511  break;
512  }
513  case kTIME:
514  case kTIMESTAMP:
515  case kDATE: {
516  if (!initialized) {
517  elem_min.bigintval = 1;
518  elem_max.bigintval = 0;
519  }
520  if (array.is_null || array.length == 0) {
521  break;
522  }
523  const auto tm_array = reinterpret_cast<int64_t*>(array.pointer);
524  for (size_t i = 0; i < array.length / sizeof(int64_t); i++) {
525  if (tm_array[i] == NULL_BIGINT) {
526  has_nulls = true;
527  } else if (initialized) {
528  elem_min.bigintval = std::min(elem_min.bigintval, tm_array[i]);
529  elem_max.bigintval = std::max(elem_max.bigintval, tm_array[i]);
530  } else {
531  elem_min.bigintval = tm_array[i];
532  elem_max.bigintval = tm_array[i];
533  initialized = true;
534  }
535  }
536  break;
537  }
538  case kCHAR:
539  case kVARCHAR:
540  case kTEXT: {
542  if (!initialized) {
543  elem_min.intval = 1;
544  elem_max.intval = 0;
545  }
546  if (array.is_null || array.length == 0) {
547  break;
548  }
549  const int32_t* int_array = (int32_t*)array.pointer;
550  for (size_t i = 0; i < array.length / sizeof(int32_t); i++) {
551  if (int_array[i] == NULL_INT) {
552  has_nulls = true;
553  } else if (initialized) {
554  elem_min.intval = std::min(elem_min.intval, int_array[i]);
555  elem_max.intval = std::max(elem_max.intval, int_array[i]);
556  } else {
557  elem_min.intval = int_array[i];
558  elem_max.intval = int_array[i];
559  initialized = true;
560  }
561  }
562  break;
563  }
564  default:
565  UNREACHABLE();
566  }
567  };
568 
569  private:
570  ArrayDatum getArrayDatumAtIndex(const int8_t* index_data, int8_t* data, size_t index) {
571  auto array_offsets = reinterpret_cast<const ArrayOffsetT*>(index_data);
572  auto current_index = index + 1;
573  auto offset = array_offsets[current_index];
574  int64_t last_offset = array_offsets[current_index - 1];
575  size_t array_byte_size = std::abs(offset) - std::abs(last_offset);
576  bool is_null = offset < 0;
577  auto current_data = data + std::abs(last_offset);
578  return is_null ? ArrayDatum(0, nullptr, true, DoNothingDeleter{})
579  : ArrayDatum(array_byte_size, current_data, false, DoNothingDeleter{});
580  }
581 
582 }; // class ArrayNoneEncoder
583 
584 #endif // ARRAY_NONE_ENCODER_H
int8_t tinyintval
Definition: sqltypes.h:212
void update_elem_stats(const ArrayDatum &array)
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
#define CHECK_EQ(x, y)
Definition: Logger.h:219
size_t getNumElemsForBytesInsertData(const std::vector< ArrayDatum > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)
#define NULL_DOUBLE
size_t num_elems_
Definition: Encoder.h:289
Definition: sqltypes.h:49
std::shared_ptr< ChunkMetadata > appendData(const std::vector< ArrayDatum > *srcData, const int start_idx, const size_t numAppendElems, const bool replicating)
#define NULL_FLOAT
#define MAX_INPUT_BUF_SIZE
Definition: Encoder.h:36
void updateStats(const double, const bool) override
#define NULL_BIGINT
std::mutex EncoderMutex_
size_t getNumElemsForBytesEncodedData(const int8_t *index_data, const int start_idx, const size_t num_elements, const size_t byte_limit) override
void updateStats(const int8_t *const src_data, const size_t num_elements) override
int8_t boolval
Definition: sqltypes.h:211
#define UNREACHABLE()
Definition: Logger.h:255
bool has_nulls
Definition: ChunkMetadata.h:28
int32_t intval
Definition: sqltypes.h:214
#define NULL_INT
virtual void read(int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:227
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:208
float floatval
Definition: sqltypes.h:216
CONSTEXPR DEVICE bool is_null(const T &value)
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:291
bool DatumEqual(const Datum a, const Datum b, const SQLTypeInfo &ti)
Definition: Datum.cpp:342
void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
int count
std::shared_ptr< ChunkMetadata > appendEncodedDataAtIndices(const int8_t *index_data, int8_t *data, const std::vector< size_t > &selected_idx) override
void updateStats(const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements) override
void resetChunkStats() override
int64_t bigintval
Definition: sqltypes.h:215
size_t getNumElems() const
Definition: Encoder.h:285
bool resetChunkStats(const ChunkStats &stats) override
: Reset chunk level stats (min, max, nulls) using new values from the argument.
int16_t smallintval
Definition: sqltypes.h:213
An AbstractBuffer is a unit of data management for a data manager.
ArrayNoneEncoder(AbstractBuffer *buffer)
#define NULL_BOOLEAN
void setIndexBuffer(AbstractBuffer *buf)
AbstractBuffer * index_buf
Definition: sqltypes.h:52
Definition: sqltypes.h:53
void readMetadata(FILE *f) override
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
static constexpr size_t DEFAULT_NULL_PADDING_SIZE
int32_t ArrayOffsetT
Definition: sqltypes.h:1091
std::shared_ptr< ChunkMetadata > appendEncodedData(const int8_t *index_data, int8_t *data, const size_t start_idx, const size_t num_elements) override
std::shared_ptr< ChunkMetadata > appendData(int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1) override
std::shared_ptr< ChunkMetadata > getMetadata(const SQLTypeInfo &ti) override
ArrayDatum getArrayDatumAtIndex(const int8_t *index_data, int8_t *data, size_t index)
Definition: sqltypes.h:41
virtual void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
SQLTypeInfo getSqlType() const
#define NULL_TINYINT
void reduceStats(const Encoder &) override
bool g_enable_watchdog false
Definition: Execute.cpp:77
ArrayOffsetT last_offset
#define CHECK(condition)
Definition: Logger.h:211
#define NULL_SMALLINT
char * f
void writeMetadata(FILE *f) override
Definition: sqltypes.h:45
constexpr double n
Definition: Utm.h:38
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:861
virtual void reserve(size_t num_bytes)=0
AbstractBuffer * getIndexBuf() const
void updateStats(const int64_t, const bool) override
void copyMetadata(const Encoder *copyFromEncoder) override
void updateStats(const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements) override
double doubleval
Definition: sqltypes.h:217