OmniSciDB  ba1bac9284
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
NoneEncoder< T > Class Template Reference

#include <NoneEncoder.h>

+ Inheritance diagram for NoneEncoder< T >:
+ Collaboration diagram for NoneEncoder< T >:

Public Member Functions

 NoneEncoder (Data_Namespace::AbstractBuffer *buffer)
 
std::shared_ptr< ChunkMetadataappendData (int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &, const bool replicating=false, const int64_t offset=-1) override
 
void getMetadata (const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
 
std::shared_ptr< ChunkMetadatagetMetadata (const SQLTypeInfo &ti) override
 
void updateStats (const int64_t val, const bool is_null) override
 
void updateStats (const double val, const bool is_null) override
 
void updateStats (const int8_t *const src_data, const size_t num_elements) override
 
void updateStatsEncoded (const int8_t *const dst_data, const size_t num_elements) override
 
void updateStats (const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements) override
 
void updateStats (const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements) override
 
void reduceStats (const Encoder &that) override
 
void writeMetadata (FILE *f) override
 
void readMetadata (FILE *f) override
 
bool resetChunkStats (const ChunkStats &stats) override
 : Reset chunk level stats (min, max, nulls) using new values from the argument. More...
 
void copyMetadata (const Encoder *copyFromEncoder) override
 
void resetChunkStats () override
 
- Public Member Functions inherited from Encoder
 Encoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~Encoder ()
 
size_t getNumElems () const
 
void setNumElems (const size_t num_elems)
 

Public Attributes

dataMin
 
dataMax
 
bool has_nulls
 

Private Member Functions

validateDataAndUpdateStats (const T &unencoded_data)
 

Additional Inherited Members

- Static Public Member Functions inherited from Encoder
static EncoderCreate (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
 
- Protected Attributes inherited from Encoder
size_t num_elems_
 
Data_Namespace::AbstractBufferbuffer_
 
DecimalOverflowValidator decimal_overflow_validator_
 
DateDaysOverflowValidator date_days_overflow_validator_
 

Detailed Description

template<typename T>
class NoneEncoder< T >

Definition at line 36 of file NoneEncoder.h.

Constructor & Destructor Documentation

template<typename T>
NoneEncoder< T >::NoneEncoder ( Data_Namespace::AbstractBuffer buffer)
inline

Definition at line 38 of file NoneEncoder.h.

References NoneEncoder< T >::resetChunkStats().

38  : Encoder(buffer) {
40  }
void resetChunkStats() override
Definition: NoneEncoder.h:212
Encoder(Data_Namespace::AbstractBuffer *buffer)
Definition: Encoder.cpp:221

+ Here is the call graph for this function:

Member Function Documentation

template<typename T>
std::shared_ptr<ChunkMetadata> NoneEncoder< T >::appendData ( int8_t *&  src_data,
const size_t  num_elems_to_append,
const SQLTypeInfo ti,
const bool  replicating = false,
const int64_t  offset = -1 
)
inlineoverridevirtual

Append data to the chunk buffer backing this encoder.

Parameters
src_dataSource data for the append
num_elems_to_appendNumber of elements to append
tiSQL Type Info for the column TODO(adb): used?
replicatingPass one value and fill the chunk with it
offsetWrite data starting at a given offset. Default is -1 which indicates an append, an offset of 0 rewrites the chunk up to num_elems_to_append.

Implements Encoder.

Definition at line 42 of file NoneEncoder.h.

References Data_Namespace::AbstractBuffer::append(), Encoder::buffer_, CHECK, CHECK_GE, NoneEncoder< T >::getMetadata(), i, Encoder::num_elems_, NoneEncoder< T >::resetChunkStats(), omnisci.dtypes::T, NoneEncoder< T >::validateDataAndUpdateStats(), and Data_Namespace::AbstractBuffer::write().

46  {
47  if (offset == 0 && num_elems_to_append >= num_elems_) {
49  }
50  T* unencodedData = reinterpret_cast<T*>(src_data);
51  std::vector<T> encoded_data;
52  if (replicating) {
53  encoded_data.resize(num_elems_to_append);
54  }
55  for (size_t i = 0; i < num_elems_to_append; ++i) {
56  size_t ri = replicating ? 0 : i;
57  T data = validateDataAndUpdateStats(unencodedData[ri]);
58  if (replicating) {
59  encoded_data[i] = data;
60  }
61  }
62  if (offset == -1) {
63  num_elems_ += num_elems_to_append;
64  buffer_->append(
65  replicating ? reinterpret_cast<int8_t*>(encoded_data.data()) : src_data,
66  num_elems_to_append * sizeof(T));
67  if (!replicating) {
68  src_data += num_elems_to_append * sizeof(T);
69  }
70  } else {
71  num_elems_ = offset + num_elems_to_append;
72  CHECK(!replicating);
73  CHECK_GE(offset, 0);
74  buffer_->write(
75  src_data, num_elems_to_append * sizeof(T), static_cast<size_t>(offset));
76  }
77  auto chunk_metadata = std::make_shared<ChunkMetadata>();
78  getMetadata(chunk_metadata);
79  return chunk_metadata;
80  }
size_t num_elems_
Definition: Encoder.h:237
#define CHECK_GE(x, y)
Definition: Logger.h:219
void resetChunkStats() override
Definition: NoneEncoder.h:212
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:239
void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
Definition: NoneEncoder.h:82
virtual void write(int8_t *src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int src_device_id=-1)=0
virtual void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
#define CHECK(condition)
Definition: Logger.h:206
T validateDataAndUpdateStats(const T &unencoded_data)
Definition: NoneEncoder.h:223

+ Here is the call graph for this function:

template<typename T>
void NoneEncoder< T >::copyMetadata ( const Encoder copyFromEncoder)
inlineoverridevirtual

Implements Encoder.

Definition at line 204 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::getNumElems(), NoneEncoder< T >::has_nulls, and Encoder::num_elems_.

204  {
205  num_elems_ = copyFromEncoder->getNumElems();
206  auto castedEncoder = reinterpret_cast<const NoneEncoder<T>*>(copyFromEncoder);
207  dataMin = castedEncoder->dataMin;
208  dataMax = castedEncoder->dataMax;
209  has_nulls = castedEncoder->has_nulls;
210  }
size_t num_elems_
Definition: Encoder.h:237
size_t getNumElems() const
Definition: Encoder.h:233
bool has_nulls
Definition: NoneEncoder.h:220

+ Here is the call graph for this function:

template<typename T>
void NoneEncoder< T >::getMetadata ( const std::shared_ptr< ChunkMetadata > &  chunkMetadata)
inlineoverridevirtual

Reimplemented from Encoder.

Definition at line 82 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::getMetadata(), and NoneEncoder< T >::has_nulls.

Referenced by NoneEncoder< T >::appendData().

82  {
83  Encoder::getMetadata(chunkMetadata); // call on parent class
84  chunkMetadata->fillChunkStats(dataMin, dataMax, has_nulls);
85  }
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:227
bool has_nulls
Definition: NoneEncoder.h:220

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T>
std::shared_ptr<ChunkMetadata> NoneEncoder< T >::getMetadata ( const SQLTypeInfo ti)
inlineoverridevirtual

Implements Encoder.

Definition at line 88 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, and NoneEncoder< T >::has_nulls.

88  {
89  auto chunk_metadata = std::make_shared<ChunkMetadata>(ti, 0, 0, ChunkStats{});
90  chunk_metadata->fillChunkStats(dataMin, dataMax, has_nulls);
91  return chunk_metadata;
92  }
bool has_nulls
Definition: NoneEncoder.h:220
template<typename T>
void NoneEncoder< T >::readMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 182 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, Encoder::num_elems_, and omnisci.dtypes::T.

182  {
183  // assumes pointer is already in right place
184  fread((int8_t*)&num_elems_, sizeof(size_t), 1, f);
185  fread((int8_t*)&dataMin, sizeof(T), 1, f);
186  fread((int8_t*)&dataMax, sizeof(T), 1, f);
187  fread((int8_t*)&has_nulls, sizeof(bool), 1, f);
188  }
size_t num_elems_
Definition: Encoder.h:237
bool has_nulls
Definition: NoneEncoder.h:220
char * f
template<typename T>
void NoneEncoder< T >::reduceStats ( const Encoder that)
inlineoverridevirtual

Implements Encoder.

Definition at line 165 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, and NoneEncoder< T >::has_nulls.

165  {
166  const auto that_typed = static_cast<const NoneEncoder&>(that);
167  if (that_typed.has_nulls) {
168  has_nulls = true;
169  }
170  dataMin = std::min(dataMin, that_typed.dataMin);
171  dataMax = std::max(dataMax, that_typed.dataMax);
172  }
bool has_nulls
Definition: NoneEncoder.h:220
template<typename T>
bool NoneEncoder< T >::resetChunkStats ( const ChunkStats )
inlineoverridevirtual

: Reset chunk level stats (min, max, nulls) using new values from the argument.

Returns
: True if an update occurred and the chunk needs to be flushed. False otherwise. Default false if metadata update is unsupported. Only reset chunk stats if the incoming stats differ from the current stats.

Reimplemented from Encoder.

Definition at line 190 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, ChunkStats::has_nulls, NoneEncoder< T >::has_nulls, ChunkStats::max, and ChunkStats::min.

190  {
191  const auto new_min = DatumFetcher::getDatumVal<T>(stats.min);
192  const auto new_max = DatumFetcher::getDatumVal<T>(stats.max);
193 
194  if (dataMin == new_min && dataMax == new_max && has_nulls == stats.has_nulls) {
195  return false;
196  }
197 
198  dataMin = new_min;
199  dataMax = new_max;
200  has_nulls = stats.has_nulls;
201  return true;
202  }
bool has_nulls
Definition: NoneEncoder.h:220
template<typename T>
void NoneEncoder< T >::resetChunkStats ( )
inlineoverridevirtual

Resets chunk metadata stats to their default values.

Implements Encoder.

Definition at line 212 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, and NoneEncoder< T >::has_nulls.

Referenced by NoneEncoder< T >::appendData(), and NoneEncoder< T >::NoneEncoder().

212  {
213  dataMin = std::numeric_limits<T>::max();
214  dataMax = std::numeric_limits<T>::lowest();
215  has_nulls = false;
216  }
bool has_nulls
Definition: NoneEncoder.h:220

+ Here is the caller graph for this function:

template<typename T>
void NoneEncoder< T >::updateStats ( const int64_t  val,
const bool  is_null 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 95 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, and omnisci.dtypes::T.

95  {
96  if (is_null) {
97  has_nulls = true;
98  } else {
99  const auto data = static_cast<T>(val);
100  dataMin = std::min(dataMin, data);
101  dataMax = std::max(dataMax, data);
102  }
103  }
CONSTEXPR DEVICE bool is_null(const T &value)
bool has_nulls
Definition: NoneEncoder.h:220
template<typename T>
void NoneEncoder< T >::updateStats ( const double  val,
const bool  is_null 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 106 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, and omnisci.dtypes::T.

106  {
107  if (is_null) {
108  has_nulls = true;
109  } else {
110  const auto data = static_cast<T>(val);
111  dataMin = std::min(dataMin, data);
112  dataMax = std::max(dataMax, data);
113  }
114  }
CONSTEXPR DEVICE bool is_null(const T &value)
bool has_nulls
Definition: NoneEncoder.h:220
template<typename T>
void NoneEncoder< T >::updateStats ( const int8_t *const  src_data,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for data without appending.

Parameters
src_data- the data with which to update statistics
num_elements- the number of elements to scan in the data

Implements Encoder.

Definition at line 116 of file NoneEncoder.h.

References i, omnisci.dtypes::T, and NoneEncoder< T >::validateDataAndUpdateStats().

116  {
117  const T* unencoded_data = reinterpret_cast<const T*>(src_data);
118  for (size_t i = 0; i < num_elements; ++i) {
119  validateDataAndUpdateStats(unencoded_data[i]);
120  }
121  }
T validateDataAndUpdateStats(const T &unencoded_data)
Definition: NoneEncoder.h:223

+ Here is the call graph for this function:

template<typename T>
void NoneEncoder< T >::updateStats ( const std::vector< std::string > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for string data without appending.

Parameters
src_data- the string data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the string data

Implements Encoder.

Definition at line 152 of file NoneEncoder.h.

References UNREACHABLE.

154  {
155  UNREACHABLE();
156  }
#define UNREACHABLE()
Definition: Logger.h:250
template<typename T>
void NoneEncoder< T >::updateStats ( const std::vector< ArrayDatum > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for array data without appending.

Parameters
src_data- the array data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the array data

Implements Encoder.

Definition at line 158 of file NoneEncoder.h.

References UNREACHABLE.

160  {
161  UNREACHABLE();
162  }
#define UNREACHABLE()
Definition: Logger.h:250
template<typename T>
void NoneEncoder< T >::updateStatsEncoded ( const int8_t *const  dst_data,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for encoded data without appending.

Parameters
dst_data- the data with which to update statistics
num_elements- the number of elements to scan in the data

Reimplemented from Encoder.

Definition at line 123 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::decimal_overflow_validator_, NoneEncoder< T >::has_nulls, i, logger::init(), omnisci.dtypes::T, and DecimalOverflowValidator::validate().

124  {
125  const T* data = reinterpret_cast<const T*>(dst_data);
126 
127  std::tie(dataMin, dataMax, has_nulls) = tbb::parallel_reduce(
128  tbb::blocked_range(size_t(0), num_elements),
129  std::tuple(dataMin, dataMax, has_nulls),
130  [&](const auto& range, auto init) {
131  auto [min, max, nulls] = init;
132  for (size_t i = range.begin(); i < range.end(); i++) {
133  if (data[i] != none_encoded_null_value<T>()) {
135  min = std::min(min, data[i]);
136  max = std::max(max, data[i]);
137  } else {
138  nulls = true;
139  }
140  }
141  return std::tuple(min, max, nulls);
142  },
143  [&](auto lhs, auto rhs) {
144  const auto [lhs_min, lhs_max, lhs_nulls] = lhs;
145  const auto [rhs_min, rhs_max, rhs_nulls] = rhs;
146  return std::tuple(std::min(lhs_min, rhs_min),
147  std::max(lhs_max, rhs_max),
148  lhs_nulls || rhs_nulls);
149  });
150  }
DecimalOverflowValidator decimal_overflow_validator_
Definition: Encoder.h:241
void init(LogOptions const &log_opts)
Definition: Logger.cpp:280
void validate(T value)
Definition: Encoder.h:54
bool has_nulls
Definition: NoneEncoder.h:220

+ Here is the call graph for this function:

template<typename T>
T NoneEncoder< T >::validateDataAndUpdateStats ( const T &  unencoded_data)
inlineprivate

Definition at line 223 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, Encoder::decimal_overflow_validator_, NoneEncoder< T >::has_nulls, and DecimalOverflowValidator::validate().

Referenced by NoneEncoder< T >::appendData(), and NoneEncoder< T >::updateStats().

223  {
224  if (unencoded_data == none_encoded_null_value<T>()) {
225  has_nulls = true;
226  } else {
227  decimal_overflow_validator_.validate(unencoded_data);
228  dataMin = std::min(dataMin, unencoded_data);
229  dataMax = std::max(dataMax, unencoded_data);
230  }
231  return unencoded_data;
232  }
DecimalOverflowValidator decimal_overflow_validator_
Definition: Encoder.h:241
void validate(T value)
Definition: Encoder.h:54
bool has_nulls
Definition: NoneEncoder.h:220

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T>
void NoneEncoder< T >::writeMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 174 of file NoneEncoder.h.

References NoneEncoder< T >::dataMax, NoneEncoder< T >::dataMin, NoneEncoder< T >::has_nulls, Encoder::num_elems_, and omnisci.dtypes::T.

174  {
175  // assumes pointer is already in right place
176  fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
177  fwrite((int8_t*)&dataMin, sizeof(T), 1, f);
178  fwrite((int8_t*)&dataMax, sizeof(T), 1, f);
179  fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
180  }
size_t num_elems_
Definition: Encoder.h:237
bool has_nulls
Definition: NoneEncoder.h:220
char * f

Member Data Documentation


The documentation for this class was generated from the following file: