OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ParquetStringNoneEncoder.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include "LazyParquetChunkLoader.h"
20 #include "ParquetEncoder.h"
21 
22 #include <parquet/schema.h>
23 #include <parquet/types.h>
24 
25 namespace foreign_storage {
26 
28  public:
30  Data_Namespace::AbstractBuffer* index_buffer)
31  : ParquetEncoder(buffer)
32  , index_buffer_(index_buffer)
33  , encode_buffer_(LazyParquetChunkLoader::batch_reader_num_elements *
34  sizeof(StringOffsetT)) {}
35 
36  void appendData(const int16_t* def_levels,
37  const int16_t* rep_levels,
38  const int64_t values_read,
39  const int64_t levels_read,
40  int8_t* values) override {
41  CHECK(levels_read > 0);
43 
44  auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
45  auto offsets = reinterpret_cast<StringOffsetT*>(encode_buffer_.data());
46  auto last_offset = buffer_->size();
47 
48  size_t total_len = 0;
49  for (int64_t i = 0, j = 0; i < levels_read; ++i) {
50  if (def_levels[i]) {
51  CHECK(j < values_read);
52  auto& byte_array = parquet_data_ptr[j++];
54  // no-op, or effectively inserting a null: total_len += 0;
55  } else {
56  total_len += byte_array.len;
57  }
58  }
59  offsets[i] = last_offset + total_len;
60  }
61  index_buffer_->append(encode_buffer_.data(), levels_read * sizeof(StringOffsetT));
62 
63  encode_buffer_.resize(std::max<size_t>(total_len, encode_buffer_.size()));
64  buffer_->reserve(total_len);
65  total_len = 0;
66  for (int64_t i = 0, j = 0; i < levels_read; ++i) {
67  if (def_levels[i]) {
68  CHECK(j < values_read);
69  auto& byte_array = parquet_data_ptr[j++];
72  i);
73  } else {
74  memcpy(encode_buffer_.data() + total_len, byte_array.ptr, byte_array.len);
75  total_len += byte_array.len;
76  }
77  } else if (is_error_tracking_enabled_ &&
79  .get_notnull()) { // item is null for NOT NULL column
81  i);
82  }
83  }
86  }
87  buffer_->append(encode_buffer_.data(), total_len);
88  }
89 
90  void appendDataTrackErrors(const int16_t* def_levels,
91  const int16_t* rep_levels,
92  const int64_t values_read,
93  const int64_t levels_read,
94  int8_t* values) override {
96  appendData(def_levels, rep_levels, values_read, levels_read, values);
97  }
98 
99  private:
101  if (!index_buffer_->size()) {
102  // write the initial starting offset
103  StringOffsetT zero = 0;
104  index_buffer_->append(reinterpret_cast<int8_t*>(&zero), sizeof(StringOffsetT));
105  }
106  }
107 
109  std::vector<int8_t> encode_buffer_;
110 };
111 
112 } // namespace foreign_storage
void appendDataTrackErrors(const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override
Data_Namespace::AbstractBuffer * index_buffer_
RejectedRowIndices invalid_indices_
int32_t StringOffsetT
Definition: sqltypes.h:1113
void appendData(const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override
An AbstractBuffer is a unit of data management for a data manager.
virtual void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
ParquetStringNoneEncoder(Data_Namespace::AbstractBuffer *buffer, Data_Namespace::AbstractBuffer *index_buffer)
#define CHECK(condition)
Definition: Logger.h:222
static constexpr size_t MAX_STRLEN
Data_Namespace::AbstractBuffer * buffer_
virtual void reserve(size_t num_bytes)=0