17 #ifndef NONE_ENCODER_H
18 #define NONE_ENCODER_H
25 #include <tbb/parallel_for.h>
26 #include <tbb/parallel_reduce.h>
31 return std::is_integral<T>::value ? inline_int_null_value<T>()
32 : inline_fp_null_value<T>();
42 std::shared_ptr<ChunkMetadata>
appendData(int8_t*& src_data,
43 const size_t num_elems_to_append,
45 const bool replicating =
false,
46 const int64_t offset = -1)
override {
47 if (offset == 0 && num_elems_to_append >=
num_elems_) {
50 T* unencodedData =
reinterpret_cast<T*
>(src_data);
51 std::vector<T> encoded_data;
53 encoded_data.resize(num_elems_to_append);
55 for (
size_t i = 0;
i < num_elems_to_append; ++
i) {
56 size_t ri = replicating ? 0 :
i;
59 encoded_data[
i] = data;
65 replicating ? reinterpret_cast<int8_t*>(encoded_data.data()) : src_data,
66 num_elems_to_append *
sizeof(
T));
68 src_data += num_elems_to_append *
sizeof(
T);
75 src_data, num_elems_to_append *
sizeof(
T), static_cast<size_t>(offset));
77 auto chunk_metadata = std::make_shared<ChunkMetadata>();
79 return chunk_metadata;
82 void getMetadata(
const std::shared_ptr<ChunkMetadata>& chunkMetadata)
override {
89 auto chunk_metadata = std::make_shared<ChunkMetadata>(ti, 0, 0,
ChunkStats{});
91 return chunk_metadata;
99 const auto data =
static_cast<T>(val);
110 const auto data =
static_cast<T>(val);
116 void updateStats(
const int8_t*
const src_data,
const size_t num_elements)
override {
117 const T* unencoded_data =
reinterpret_cast<const T*
>(src_data);
118 for (
size_t i = 0;
i < num_elements; ++
i) {
124 const size_t num_elements)
override {
125 const T* data =
reinterpret_cast<const T*
>(dst_data);
128 tbb::blocked_range(
size_t(0), num_elements),
130 [&](
const auto& range,
auto init) {
131 auto [min, max, nulls] =
init;
132 for (
size_t i = range.begin();
i < range.end();
i++) {
133 if (data[
i] != none_encoded_null_value<T>()) {
135 min = std::min(min, data[i]);
136 max = std::max(max, data[i]);
141 return std::tuple(min, max, nulls);
143 [&](
auto lhs,
auto rhs) {
144 const auto [lhs_min, lhs_max, lhs_nulls] = lhs;
145 const auto [rhs_min, rhs_max, rhs_nulls] = rhs;
146 return std::tuple(std::min(lhs_min, rhs_min),
147 std::max(lhs_max, rhs_max),
148 lhs_nulls || rhs_nulls);
153 const size_t start_idx,
154 const size_t num_elements)
override {
159 const size_t start_idx,
160 const size_t num_elements)
override {
166 const auto that_typed =
static_cast<const NoneEncoder&
>(that);
167 if (that_typed.has_nulls) {
176 fwrite((int8_t*)&
num_elems_,
sizeof(
size_t), 1, f);
177 fwrite((int8_t*)&
dataMin,
sizeof(
T), 1, f);
178 fwrite((int8_t*)&
dataMax,
sizeof(
T), 1, f);
179 fwrite((int8_t*)&
has_nulls,
sizeof(
bool), 1, f);
184 fread((int8_t*)&
num_elems_,
sizeof(
size_t), 1, f);
185 fread((int8_t*)&
dataMin,
sizeof(
T), 1, f);
186 fread((int8_t*)&
dataMax,
sizeof(
T), 1, f);
187 fread((int8_t*)&
has_nulls,
sizeof(
bool), 1, f);
191 const auto new_min = DatumFetcher::getDatumVal<T>(stats.
min);
192 const auto new_max = DatumFetcher::getDatumVal<T>(stats.
max);
206 auto castedEncoder =
reinterpret_cast<const NoneEncoder<T>*
>(copyFromEncoder);
207 dataMin = castedEncoder->dataMin;
208 dataMax = castedEncoder->dataMax;
213 dataMin = std::numeric_limits<T>::max();
214 dataMax = std::numeric_limits<T>::lowest();
224 if (unencoded_data == none_encoded_null_value<T>()) {
231 return unencoded_data;
235 #endif // NONE_ENCODER_H
void updateStats(const int8_t *const src_data, const size_t num_elements) override
void writeMetadata(FILE *f) override
DecimalOverflowValidator decimal_overflow_validator_
void updateStats(const int64_t val, const bool is_null) override
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
void resetChunkStats() override
void updateStats(const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements) override
CONSTEXPR DEVICE bool is_null(const T &value)
Data_Namespace::AbstractBuffer * buffer_
std::shared_ptr< ChunkMetadata > appendData(int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &, const bool replicating=false, const int64_t offset=-1) override
void init(LogOptions const &log_opts)
size_t getNumElems() const
An AbstractBuffer is a unit of data management for a data manager.
void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
virtual void write(int8_t *src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int src_device_id=-1)=0
T none_encoded_null_value()
bool resetChunkStats(const ChunkStats &stats) override
: Reset chunk level stats (min, max, nulls) using new values from the argument.
void updateStats(const double val, const bool is_null) override
void updateStats(const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements) override
std::shared_ptr< ChunkMetadata > getMetadata(const SQLTypeInfo &ti) override
virtual void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
NoneEncoder(Data_Namespace::AbstractBuffer *buffer)
void updateStatsEncoded(const int8_t *const dst_data, const size_t num_elements) override
void reduceStats(const Encoder &that) override
void copyMetadata(const Encoder *copyFromEncoder) override
T validateDataAndUpdateStats(const T &unencoded_data)
void readMetadata(FILE *f) override