OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ArrayNoneEncoder.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #ifndef ARRAY_NONE_ENCODER_H
24 #define ARRAY_NONE_ENCODER_H
25 
26 #include "Logger/Logger.h"
27 
28 #include <cassert>
29 #include <cstring>
30 #include <memory>
31 #include <mutex>
32 #include <string>
33 #include <vector>
34 #include "AbstractBuffer.h"
35 #include "ChunkMetadata.h"
36 #include "Encoder.h"
37 
39 
40 // TODO(Misiu): All of these functions should be moved to a .cpp file.
41 class ArrayNoneEncoder : public Encoder {
42  public:
44  : Encoder(buffer)
45  , has_nulls(false)
47  , index_buf(nullptr)
48  , last_offset(-1) {}
49 
50  size_t getNumElemsForBytesInsertData(const std::vector<ArrayDatum>* srcData,
51  const int start_idx,
52  const size_t numAppendElems,
53  const size_t byteLimit,
54  const bool replicating = false) {
55  size_t dataSize = 0;
56 
57  size_t n = start_idx;
58  for (; n < start_idx + numAppendElems; n++) {
59  size_t len = (*srcData)[replicating ? 0 : n].length;
60  if (dataSize + len > byteLimit) {
61  break;
62  }
63  dataSize += len;
64  }
65  return n - start_idx;
66  }
67 
68  size_t getNumElemsForBytesEncodedDataAtIndices(const int8_t* index_data,
69  const std::vector<size_t>& selected_idx,
70  const size_t byte_limit) override {
71  size_t num_elements = 0;
72  size_t data_size = 0;
73  for (const auto& offset_index : selected_idx) {
74  auto element_size = getArrayDatumSizeAtIndex(index_data, offset_index);
75  if (data_size + element_size > byte_limit) {
76  break;
77  }
78  data_size += element_size;
79  num_elements++;
80  }
81  return num_elements;
82  }
83 
84  std::shared_ptr<ChunkMetadata> appendData(int8_t*& src_data,
85  const size_t num_elems_to_append,
86  const SQLTypeInfo& ti,
87  const bool replicating = false,
88  const int64_t offset = -1) override {
89  UNREACHABLE(); // should never be called for arrays
90  return nullptr;
91  }
92 
93  std::shared_ptr<ChunkMetadata> appendEncodedDataAtIndices(
94  const int8_t* index_data,
95  int8_t* data,
96  const std::vector<size_t>& selected_idx) override {
97  std::vector<ArrayDatum> data_subset;
98  data_subset.reserve(selected_idx.size());
99  for (const auto& offset_index : selected_idx) {
100  data_subset.emplace_back(getArrayDatumAtIndex(index_data, data, offset_index));
101  }
102  return appendData(&data_subset, 0, selected_idx.size(), false);
103  }
104 
105  std::shared_ptr<ChunkMetadata> appendEncodedData(const int8_t* index_data,
106  int8_t* data,
107  const size_t start_idx,
108  const size_t num_elements) override {
109  std::vector<ArrayDatum> data_subset;
110  data_subset.reserve(num_elements);
111  for (size_t count = 0; count < num_elements; ++count) {
112  auto current_index = start_idx + count;
113  data_subset.emplace_back(getArrayDatumAtIndex(index_data, data, current_index));
114  }
115  return appendData(&data_subset, 0, num_elements, false);
116  }
117 
118  std::shared_ptr<ChunkMetadata> appendData(const std::vector<ArrayDatum>* srcData,
119  const int start_idx,
120  const size_t numAppendElems,
121  const bool replicating) {
122  CHECK(index_buf != nullptr); // index_buf must be set before this.
123  size_t append_index_size = numAppendElems * sizeof(ArrayOffsetT);
124  if (num_elems_ == 0) {
125  append_index_size += sizeof(ArrayOffsetT); // plus one for the initial offset
126  }
127  index_buf->reserve(index_buf->size() + append_index_size);
128 
129  bool first_elem_padded = false;
130  ArrayOffsetT initial_offset = 0;
131  if (num_elems_ == 0) {
132  if ((*srcData)[0].is_null || (*srcData)[0].length <= 1) {
133  // Covers following potentially problematic first arrays:
134  // (1) NULL array, issue - can't encode a NULL with 0 initial offset
135  // otherwise, if first array is not NULL:
136  // (2) length=1 array - could be followed by a {}*/NULL, covers tinyint,bool
137  // (3) empty array - could be followed by {}*/NULL, or {}*|{x}|{}*|NULL, etc.
138  initial_offset = DEFAULT_NULL_PADDING_SIZE;
139  first_elem_padded = true;
140  }
141  index_buf->append((int8_t*)&initial_offset,
142  sizeof(ArrayOffsetT)); // write the initial offset
143  last_offset = initial_offset;
144  } else {
145  // Valid last_offset is never negative
146  // always need to read a valid last offset from buffer/disk
147  // b/c now due to vacuum "last offset" may go backward and if
148  // index chunk was not reloaded last_offset would go way off!
149  index_buf->read((int8_t*)&last_offset,
150  sizeof(ArrayOffsetT),
151  index_buf->size() - sizeof(ArrayOffsetT),
153  CHECK(last_offset != -1);
154  // If the loaded offset is negative it means the last value was a NULL array,
155  // convert to a valid last offset
156  if (last_offset < 0) {
158  }
159  }
160  // Need to start data from 8 byte offset if first array encoded is a NULL array
161  size_t append_data_size = (first_elem_padded) ? DEFAULT_NULL_PADDING_SIZE : 0;
162  for (size_t n = start_idx; n < start_idx + numAppendElems; n++) {
163  // NULL arrays don't take any space so don't add to the data size
164  if ((*srcData)[replicating ? 0 : n].is_null) {
165  continue;
166  }
167  append_data_size += (*srcData)[replicating ? 0 : n].length;
168  }
169  buffer_->reserve(buffer_->size() + append_data_size);
170 
171  size_t inbuf_size = std::min(std::max(append_index_size, append_data_size),
172  (size_t)MAX_INPUT_BUF_SIZE);
173  auto gc_inbuf = std::make_unique<int8_t[]>(inbuf_size);
174  auto inbuf = gc_inbuf.get();
175  for (size_t num_appended = 0; num_appended < numAppendElems;) {
176  ArrayOffsetT* p = (ArrayOffsetT*)inbuf;
177  size_t i;
178  for (i = 0; num_appended < numAppendElems && i < inbuf_size / sizeof(ArrayOffsetT);
179  i++, num_appended++) {
180  p[i] =
181  last_offset + (*srcData)[replicating ? 0 : num_appended + start_idx].length;
182  last_offset = p[i];
183  if ((*srcData)[replicating ? 0 : num_appended + start_idx].is_null) {
184  // Record array NULLness in the index buffer
185  p[i] = -p[i];
186  }
187  }
188  index_buf->append(inbuf, i * sizeof(ArrayOffsetT));
189  }
190 
191  // Pad buffer_ with 8 bytes if first encoded array is a NULL array
192  if (first_elem_padded) {
193  auto padding_size = DEFAULT_NULL_PADDING_SIZE;
194  buffer_->append(inbuf, padding_size);
195  }
196  for (size_t num_appended = 0; num_appended < numAppendElems;) {
197  size_t size = 0;
198  for (int i = start_idx + num_appended;
199  num_appended < numAppendElems && size < inbuf_size;
200  i++, num_appended++) {
201  if ((*srcData)[replicating ? 0 : i].is_null) {
202  continue; // NULL arrays don't take up any space in the data buffer
203  }
204  size_t len = (*srcData)[replicating ? 0 : i].length;
205  if (len > inbuf_size) {
206  // for large strings, append on its own
207  if (size > 0) {
208  buffer_->append(inbuf, size);
209  }
210  size = 0;
211  buffer_->append((*srcData)[replicating ? 0 : i].pointer, len);
212  num_appended++;
213  break;
214  } else if (size + len > inbuf_size) {
215  break;
216  }
217  char* dest = (char*)inbuf + size;
218  if (len > 0) {
219  std::memcpy((void*)dest, (void*)(*srcData)[replicating ? 0 : i].pointer, len);
220  size += len;
221  }
222  }
223  if (size > 0) {
224  buffer_->append(inbuf, size);
225  }
226  }
227  // make sure buffer_ is flushed even if no new data is appended to it
228  // (e.g. empty strings) because the metadata needs to be flushed.
229  if (!buffer_->isDirty()) {
230  buffer_->setDirty();
231  }
232 
233  // keep Chunk statistics with array elements
234  for (size_t n = start_idx; n < start_idx + numAppendElems; n++) {
235  update_elem_stats((*srcData)[replicating ? 0 : n]);
236  }
237  num_elems_ += numAppendElems;
238  auto chunk_metadata = std::make_shared<ChunkMetadata>();
239  getMetadata(chunk_metadata);
240  return chunk_metadata;
241  }
242 
243  void getMetadata(const std::shared_ptr<ChunkMetadata>& chunkMetadata) override {
244  Encoder::getMetadata(chunkMetadata); // call on parent class
245  chunkMetadata->fillChunkStats(elem_min, elem_max, has_nulls);
246  }
247 
248  // Only called from the executor for synthesized meta-information.
249  std::shared_ptr<ChunkMetadata> getMetadata(const SQLTypeInfo& ti) override {
250  auto chunk_metadata = std::make_shared<ChunkMetadata>(
251  ti, 0, 0, ChunkStats{elem_min, elem_max, has_nulls});
252  return chunk_metadata;
253  }
254 
255  void updateStats(const int64_t, const bool) override { CHECK(false); }
256 
257  void updateStats(const double, const bool) override { CHECK(false); }
258 
259  void updateStats(const int8_t* const src_data, const size_t num_elements) override {
260  CHECK(false);
261  }
262 
263  void updateStats(const std::vector<std::string>* const src_data,
264  const size_t start_idx,
265  const size_t num_elements) override {
266  UNREACHABLE();
267  }
268 
269  void updateStats(const std::vector<ArrayDatum>* const src_data,
270  const size_t start_idx,
271  const size_t num_elements) override {
272  for (size_t n = start_idx; n < start_idx + num_elements; n++) {
273  update_elem_stats((*src_data)[n]);
274  }
275  }
276 
277  void reduceStats(const Encoder&) override { CHECK(false); }
278 
279  void writeMetadata(FILE* f) override {
280  // assumes pointer is already in right place
281  fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
282  fwrite((int8_t*)&elem_min, sizeof(Datum), 1, f);
283  fwrite((int8_t*)&elem_max, sizeof(Datum), 1, f);
284  fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
285  fwrite((int8_t*)&initialized, sizeof(bool), 1, f);
286  }
287 
288  void readMetadata(FILE* f) override {
289  // assumes pointer is already in right place
290  fread((int8_t*)&num_elems_, sizeof(size_t), 1, f);
291  fread((int8_t*)&elem_min, sizeof(Datum), 1, f);
292  fread((int8_t*)&elem_max, sizeof(Datum), 1, f);
293  fread((int8_t*)&has_nulls, sizeof(bool), 1, f);
294  fread((int8_t*)&initialized, sizeof(bool), 1, f);
295  }
296 
297  void copyMetadata(const Encoder* copyFromEncoder) override {
298  num_elems_ = copyFromEncoder->getNumElems();
299  auto array_encoder = dynamic_cast<const ArrayNoneEncoder*>(copyFromEncoder);
300  elem_min = array_encoder->elem_min;
301  elem_max = array_encoder->elem_max;
302  has_nulls = array_encoder->has_nulls;
303  initialized = array_encoder->initialized;
304  }
305 
306  AbstractBuffer* getIndexBuf() const { return index_buf; }
307 
308  bool resetChunkStats(const ChunkStats& stats) override {
309  auto elem_type = buffer_->getSqlType().get_elem_type();
310  if (initialized && DatumEqual(elem_min, stats.min, elem_type) &&
311  DatumEqual(elem_max, stats.max, elem_type) && has_nulls == stats.has_nulls) {
312  return false;
313  }
314  elem_min = stats.min;
315  elem_max = stats.max;
316  has_nulls = stats.has_nulls;
317  return true;
318  }
319 
320  void resetChunkStats() override {
321  has_nulls = false;
322  initialized = false;
323  }
324 
327  bool has_nulls;
330  std::unique_lock<std::mutex> lock(EncoderMutex_);
331  index_buf = buf;
332  }
333 
334  static constexpr size_t DEFAULT_NULL_PADDING_SIZE{8};
335 
336  private:
337  std::mutex EncoderMutex_;
340 
341  void update_elem_stats(const ArrayDatum& array) {
342  if (array.is_null) {
343  has_nulls = true;
344  }
345  switch (buffer_->getSqlType().get_subtype()) {
346  case kBOOLEAN: {
347  if (!initialized) {
348  elem_min.boolval = 1;
349  elem_max.boolval = 0;
350  }
351  if (array.is_null || array.length == 0) {
352  break;
353  }
354  const int8_t* bool_array = array.pointer;
355  for (size_t i = 0; i < array.length / sizeof(bool); i++) {
356  if (bool_array[i] == NULL_BOOLEAN) {
357  has_nulls = true;
358  } else if (initialized) {
359  elem_min.boolval = std::min(elem_min.boolval, bool_array[i]);
360  elem_max.boolval = std::max(elem_max.boolval, bool_array[i]);
361  } else {
362  elem_min.boolval = bool_array[i];
363  elem_max.boolval = bool_array[i];
364  initialized = true;
365  }
366  }
367  break;
368  }
369  case kINT: {
370  if (!initialized) {
371  elem_min.intval = 1;
372  elem_max.intval = 0;
373  }
374  if (array.is_null || array.length == 0) {
375  break;
376  }
377  const int32_t* int_array = (int32_t*)array.pointer;
378  for (size_t i = 0; i < array.length / sizeof(int32_t); i++) {
379  if (int_array[i] == NULL_INT) {
380  has_nulls = true;
381  } else if (initialized) {
382  elem_min.intval = std::min(elem_min.intval, int_array[i]);
383  elem_max.intval = std::max(elem_max.intval, int_array[i]);
384  } else {
385  elem_min.intval = int_array[i];
386  elem_max.intval = int_array[i];
387  initialized = true;
388  }
389  }
390  break;
391  }
392  case kSMALLINT: {
393  if (!initialized) {
394  elem_min.smallintval = 1;
395  elem_max.smallintval = 0;
396  }
397  if (array.is_null || array.length == 0) {
398  break;
399  }
400  const int16_t* int_array = (int16_t*)array.pointer;
401  for (size_t i = 0; i < array.length / sizeof(int16_t); i++) {
402  if (int_array[i] == NULL_SMALLINT) {
403  has_nulls = true;
404  } else if (initialized) {
405  elem_min.smallintval = std::min(elem_min.smallintval, int_array[i]);
406  elem_max.smallintval = std::max(elem_max.smallintval, int_array[i]);
407  } else {
408  elem_min.smallintval = int_array[i];
409  elem_max.smallintval = int_array[i];
410  initialized = true;
411  }
412  }
413  break;
414  }
415  case kTINYINT: {
416  if (!initialized) {
417  elem_min.tinyintval = 1;
418  elem_max.tinyintval = 0;
419  }
420  if (array.is_null || array.length == 0) {
421  break;
422  }
423  const int8_t* int_array = (int8_t*)array.pointer;
424  for (size_t i = 0; i < array.length / sizeof(int8_t); i++) {
425  if (int_array[i] == NULL_TINYINT) {
426  has_nulls = true;
427  } else if (initialized) {
428  elem_min.tinyintval = std::min(elem_min.tinyintval, int_array[i]);
429  elem_max.tinyintval = std::max(elem_max.tinyintval, int_array[i]);
430  } else {
431  elem_min.tinyintval = int_array[i];
432  elem_max.tinyintval = int_array[i];
433  initialized = true;
434  }
435  }
436  break;
437  }
438  case kBIGINT:
439  case kNUMERIC:
440  case kDECIMAL: {
441  if (!initialized) {
442  elem_min.bigintval = 1;
443  elem_max.bigintval = 0;
444  }
445  if (array.is_null || array.length == 0) {
446  break;
447  }
448  const int64_t* int_array = (int64_t*)array.pointer;
449  for (size_t i = 0; i < array.length / sizeof(int64_t); i++) {
450  if (int_array[i] == NULL_BIGINT) {
451  has_nulls = true;
452  } else if (initialized) {
453  elem_min.bigintval = std::min(elem_min.bigintval, int_array[i]);
454  elem_max.bigintval = std::max(elem_max.bigintval, int_array[i]);
455  } else {
456  elem_min.bigintval = int_array[i];
457  elem_max.bigintval = int_array[i];
458  initialized = true;
459  }
460  }
461  break;
462  }
463  case kFLOAT: {
464  if (!initialized) {
465  elem_min.floatval = 1.0;
466  elem_max.floatval = 0.0;
467  }
468  if (array.is_null || array.length == 0) {
469  break;
470  }
471  const float* flt_array = (float*)array.pointer;
472  for (size_t i = 0; i < array.length / sizeof(float); i++) {
473  if (flt_array[i] == NULL_FLOAT) {
474  has_nulls = true;
475  } else if (initialized) {
476  elem_min.floatval = std::min(elem_min.floatval, flt_array[i]);
477  elem_max.floatval = std::max(elem_max.floatval, flt_array[i]);
478  } else {
479  elem_min.floatval = flt_array[i];
480  elem_max.floatval = flt_array[i];
481  initialized = true;
482  }
483  }
484  break;
485  }
486  case kDOUBLE: {
487  if (!initialized) {
488  elem_min.doubleval = 1.0;
489  elem_max.doubleval = 0.0;
490  }
491  if (array.is_null || array.length == 0) {
492  break;
493  }
494  const double* dbl_array = (double*)array.pointer;
495  for (size_t i = 0; i < array.length / sizeof(double); i++) {
496  if (dbl_array[i] == NULL_DOUBLE) {
497  has_nulls = true;
498  } else if (initialized) {
499  elem_min.doubleval = std::min(elem_min.doubleval, dbl_array[i]);
500  elem_max.doubleval = std::max(elem_max.doubleval, dbl_array[i]);
501  } else {
502  elem_min.doubleval = dbl_array[i];
503  elem_max.doubleval = dbl_array[i];
504  initialized = true;
505  }
506  }
507  break;
508  }
509  case kTIME:
510  case kTIMESTAMP:
511  case kDATE: {
512  if (!initialized) {
513  elem_min.bigintval = 1;
514  elem_max.bigintval = 0;
515  }
516  if (array.is_null || array.length == 0) {
517  break;
518  }
519  const auto tm_array = reinterpret_cast<int64_t*>(array.pointer);
520  for (size_t i = 0; i < array.length / sizeof(int64_t); i++) {
521  if (tm_array[i] == NULL_BIGINT) {
522  has_nulls = true;
523  } else if (initialized) {
524  elem_min.bigintval = std::min(elem_min.bigintval, tm_array[i]);
525  elem_max.bigintval = std::max(elem_max.bigintval, tm_array[i]);
526  } else {
527  elem_min.bigintval = tm_array[i];
528  elem_max.bigintval = tm_array[i];
529  initialized = true;
530  }
531  }
532  break;
533  }
534  case kCHAR:
535  case kVARCHAR:
536  case kTEXT: {
538  if (!initialized) {
539  elem_min.intval = 1;
540  elem_max.intval = 0;
541  }
542  if (array.is_null || array.length == 0) {
543  break;
544  }
545  const int32_t* int_array = (int32_t*)array.pointer;
546  for (size_t i = 0; i < array.length / sizeof(int32_t); i++) {
547  if (int_array[i] == NULL_INT) {
548  has_nulls = true;
549  } else if (initialized) {
550  elem_min.intval = std::min(elem_min.intval, int_array[i]);
551  elem_max.intval = std::max(elem_max.intval, int_array[i]);
552  } else {
553  elem_min.intval = int_array[i];
554  elem_max.intval = int_array[i];
555  initialized = true;
556  }
557  }
558  break;
559  }
560  default:
561  UNREACHABLE();
562  }
563  };
564 
565  private:
566  std::pair<ArrayOffsetT, ArrayOffsetT> getArrayOffsetsAtIndex(const int8_t* index_data,
567  size_t index) {
568  auto array_offsets = reinterpret_cast<const ArrayOffsetT*>(index_data);
569  auto current_index = index + 1;
570  auto offset = array_offsets[current_index];
571  int64_t last_offset = array_offsets[current_index - 1];
572  return {offset, last_offset};
573  }
574 
575  size_t getArrayDatumSizeAtIndex(const int8_t* index_data, size_t index) {
576  auto [offset, last_offset] = getArrayOffsetsAtIndex(index_data, index);
577  size_t array_byte_size = std::abs(offset) - std::abs(last_offset);
578  return array_byte_size;
579  }
580 
581  ArrayDatum getArrayDatumAtIndex(const int8_t* index_data, int8_t* data, size_t index) {
582  auto [offset, last_offset] = getArrayOffsetsAtIndex(index_data, index);
583  size_t array_byte_size = std::abs(offset) - std::abs(last_offset);
584  bool is_null = offset < 0;
585  auto current_data = data + std::abs(last_offset);
586  return is_null ? ArrayDatum(0, nullptr, true, DoNothingDeleter{})
587  : ArrayDatum(array_byte_size, current_data, false, DoNothingDeleter{});
588  }
589 
590 }; // class ArrayNoneEncoder
591 
592 #endif // ARRAY_NONE_ENCODER_H
int8_t tinyintval
Definition: Datum.h:71
void update_elem_stats(const ArrayDatum &array)
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
#define CHECK_EQ(x, y)
Definition: Logger.h:301
size_t getNumElemsForBytesInsertData(const std::vector< ArrayDatum > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)
#define NULL_DOUBLE
size_t num_elems_
Definition: Encoder.h:288
Definition: sqltypes.h:76
std::shared_ptr< ChunkMetadata > appendData(const std::vector< ArrayDatum > *srcData, const int start_idx, const size_t numAppendElems, const bool replicating)
#define NULL_FLOAT
#define MAX_INPUT_BUF_SIZE
Definition: Encoder.h:36
void updateStats(const double, const bool) override
#define NULL_BIGINT
std::mutex EncoderMutex_
void updateStats(const int8_t *const src_data, const size_t num_elements) override
int8_t boolval
Definition: Datum.h:70
#define UNREACHABLE()
Definition: Logger.h:338
bool has_nulls
Definition: ChunkMetadata.h:30
dictionary stats
Definition: report.py:116
int32_t intval
Definition: Datum.h:73
#define NULL_INT
virtual void read(int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:231
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:229
size_t getNumElemsForBytesEncodedDataAtIndices(const int8_t *index_data, const std::vector< size_t > &selected_idx, const size_t byte_limit) override
float floatval
Definition: Datum.h:75
CONSTEXPR DEVICE bool is_null(const T &value)
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:290
bool DatumEqual(const Datum a, const Datum b, const SQLTypeInfo &ti)
Definition: Datum.cpp:408
void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
std::shared_ptr< ChunkMetadata > appendEncodedDataAtIndices(const int8_t *index_data, int8_t *data, const std::vector< size_t > &selected_idx) override
void updateStats(const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements) override
void resetChunkStats() override
int64_t bigintval
Definition: Datum.h:74
size_t getNumElems() const
Definition: Encoder.h:284
bool resetChunkStats(const ChunkStats &stats) override
: Reset chunk level stats (min, max, nulls) using new values from the argument.
int16_t smallintval
Definition: Datum.h:72
An AbstractBuffer is a unit of data management for a data manager.
ArrayNoneEncoder(AbstractBuffer *buffer)
#define NULL_BOOLEAN
void setIndexBuffer(AbstractBuffer *buf)
AbstractBuffer * index_buf
Definition: sqltypes.h:79
Definition: sqltypes.h:80
void readMetadata(FILE *f) override
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
static constexpr size_t DEFAULT_NULL_PADDING_SIZE
int32_t ArrayOffsetT
Definition: sqltypes.h:1494
std::shared_ptr< ChunkMetadata > appendEncodedData(const int8_t *index_data, int8_t *data, const size_t start_idx, const size_t num_elements) override
std::shared_ptr< ChunkMetadata > appendData(int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1) override
std::shared_ptr< ChunkMetadata > getMetadata(const SQLTypeInfo &ti) override
ArrayDatum getArrayDatumAtIndex(const int8_t *index_data, int8_t *data, size_t index)
torch::Tensor f(torch::Tensor x, torch::Tensor W_target, torch::Tensor b_target)
Definition: sqltypes.h:68
virtual void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
SQLTypeInfo getSqlType() const
#define NULL_TINYINT
void reduceStats(const Encoder &) override
std::pair< ArrayOffsetT, ArrayOffsetT > getArrayOffsetsAtIndex(const int8_t *index_data, size_t index)
bool g_enable_watchdog false
Definition: Execute.cpp:80
ArrayOffsetT last_offset
#define CHECK(condition)
Definition: Logger.h:291
#define NULL_SMALLINT
void writeMetadata(FILE *f) override
Definition: sqltypes.h:72
constexpr double n
Definition: Utm.h:38
Definition: Datum.h:69
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:975
virtual void reserve(size_t num_bytes)=0
AbstractBuffer * getIndexBuf() const
void updateStats(const int64_t, const bool) override
void copyMetadata(const Encoder *copyFromEncoder) override
void updateStats(const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements) override
double doubleval
Definition: Datum.h:76
size_t getArrayDatumSizeAtIndex(const int8_t *index_data, size_t index)