OmniSciDB  04ee39c94c
FixedLengthEncoder< T, V > Class Template Reference

#include <FixedLengthEncoder.h>

+ Inheritance diagram for FixedLengthEncoder< T, V >:
+ Collaboration diagram for FixedLengthEncoder< T, V >:

Public Member Functions

 FixedLengthEncoder (Data_Namespace::AbstractBuffer *buffer)
 
ChunkMetadata appendData (int8_t *&srcData, const size_t numAppendElems, const SQLTypeInfo &ti, const bool replicating=false) override
 
void getMetadata (ChunkMetadata &chunkMetadata) override
 
ChunkMetadata getMetadata (const SQLTypeInfo &ti) override
 
void updateStats (const int64_t val, const bool is_null) override
 
void updateStats (const double val, const bool is_null) override
 
void reduceStats (const Encoder &that) override
 
void copyMetadata (const Encoder *copyFromEncoder) override
 
void writeMetadata (FILE *f) override
 
void readMetadata (FILE *f) override
 
bool resetChunkStats (const ChunkStats &stats) override
 : Reset chunk level stats (min, max, nulls) using new values from the argument. More...
 
- Public Member Functions inherited from Encoder
 Encoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~Encoder ()
 
size_t getNumElems () const
 
void setNumElems (const size_t num_elems)
 

Public Attributes

dataMin
 
dataMax
 
bool has_nulls
 

Additional Inherited Members

- Static Public Member Functions inherited from Encoder
static EncoderCreate (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
 
- Protected Attributes inherited from Encoder
size_t num_elems_
 
Data_Namespace::AbstractBufferbuffer_
 
DecimalOverflowValidator decimal_overflow_validator_
 
DateDaysOverflowValidator date_days_overflow_validator_
 

Detailed Description

template<typename T, typename V>
class FixedLengthEncoder< T, V >

Definition at line 30 of file FixedLengthEncoder.h.

Constructor & Destructor Documentation

◆ FixedLengthEncoder()

template<typename T, typename V>
FixedLengthEncoder< T, V >::FixedLengthEncoder ( Data_Namespace::AbstractBuffer buffer)
inline

Definition at line 32 of file FixedLengthEncoder.h.

33  : Encoder(buffer)
34  , dataMin(std::numeric_limits<T>::max())
35  , dataMax(std::numeric_limits<T>::min())
36  , has_nulls(false) {}
Encoder(Data_Namespace::AbstractBuffer *buffer)
Definition: Encoder.cpp:221

Member Function Documentation

◆ appendData()

template<typename T, typename V>
ChunkMetadata FixedLengthEncoder< T, V >::appendData ( int8_t *&  srcData,
const size_t  numAppendElems,
const SQLTypeInfo ti,
const bool  replicating = false 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 38 of file FixedLengthEncoder.h.

References Data_Namespace::AbstractBuffer::append(), Encoder::buffer_, FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, Encoder::decimal_overflow_validator_, logger::ERROR, FixedLengthEncoder< T, V >::getMetadata(), FixedLengthEncoder< T, V >::has_nulls, LOG, Encoder::num_elems_, to_string(), and DecimalOverflowValidator::validate().

41  {
42  T* unencodedData = reinterpret_cast<T*>(srcData);
43  auto encodedData = std::make_unique<V[]>(numAppendElems);
44  for (size_t i = 0; i < numAppendElems; ++i) {
45  size_t ri = replicating ? 0 : i;
46  encodedData.get()[i] = static_cast<V>(unencodedData[ri]);
47  if (unencodedData[ri] != encodedData.get()[i]) {
48  decimal_overflow_validator_.validate(unencodedData[ri]);
49  LOG(ERROR) << "Fixed encoding failed, Unencoded: " +
50  std::to_string(unencodedData[ri]) +
51  " encoded: " + std::to_string(encodedData.get()[i]);
52  } else {
53  T data = unencodedData[ri];
54  if (data == std::numeric_limits<V>::min()) {
55  has_nulls = true;
56  } else {
58  dataMin = std::min(dataMin, data);
59  dataMax = std::max(dataMax, data);
60  }
61  }
62  }
63  num_elems_ += numAppendElems;
64 
65  // assume always CPU_BUFFER?
66  buffer_->append((int8_t*)(encodedData.get()), numAppendElems * sizeof(V));
67  ChunkMetadata chunkMetadata;
68  getMetadata(chunkMetadata);
69  if (!replicating) {
70  srcData += numAppendElems * sizeof(T);
71  }
72  return chunkMetadata;
73  }
size_t num_elems_
Definition: Encoder.h:179
DecimalOverflowValidator decimal_overflow_validator_
Definition: Encoder.h:184
#define LOG(tag)
Definition: Logger.h:182
std::string to_string(char const *&&v)
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:181
virtual void append(int8_t *src, const size_t numBytes, const MemoryLevel srcBufferType=CPU_LEVEL, const int deviceId=-1)=0
void getMetadata(ChunkMetadata &chunkMetadata) override
void validate(T value)
Definition: Encoder.h:54
+ Here is the call graph for this function:

◆ copyMetadata()

template<typename T, typename V>
void FixedLengthEncoder< T, V >::copyMetadata ( const Encoder copyFromEncoder)
inlineoverridevirtual

Implements Encoder.

Definition at line 119 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, Encoder::getNumElems(), FixedLengthEncoder< T, V >::has_nulls, and Encoder::num_elems_.

119  {
120  num_elems_ = copyFromEncoder->getNumElems();
121  auto castedEncoder =
122  reinterpret_cast<const FixedLengthEncoder<T, V>*>(copyFromEncoder);
123  dataMin = castedEncoder->dataMin;
124  dataMax = castedEncoder->dataMax;
125  has_nulls = castedEncoder->has_nulls;
126  }
size_t num_elems_
Definition: Encoder.h:179
size_t getNumElems() const
Definition: Encoder.h:175
+ Here is the call graph for this function:

◆ getMetadata() [1/2]

template<typename T, typename V>
void FixedLengthEncoder< T, V >::getMetadata ( ChunkMetadata chunkMetadata)
inlineoverridevirtual

Reimplemented from Encoder.

Definition at line 75 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, ChunkMetadata::fillChunkStats(), Encoder::getMetadata(), and FixedLengthEncoder< T, V >::has_nulls.

Referenced by FixedLengthEncoder< T, V >::appendData().

75  {
76  Encoder::getMetadata(chunkMetadata); // call on parent class
77  chunkMetadata.fillChunkStats(dataMin, dataMax, has_nulls);
78  }
void fillChunkStats(const T min, const T max, const bool has_nulls)
Definition: ChunkMetadata.h:38
virtual void getMetadata(ChunkMetadata &chunkMetadata)
Definition: Encoder.cpp:227
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadata() [2/2]

template<typename T, typename V>
ChunkMetadata FixedLengthEncoder< T, V >::getMetadata ( const SQLTypeInfo ti)
inlineoverridevirtual

Implements Encoder.

Definition at line 81 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, and FixedLengthEncoder< T, V >::has_nulls.

81  {
82  ChunkMetadata chunk_metadata{ti, 0, 0, ChunkStats{}};
83  chunk_metadata.fillChunkStats(dataMin, dataMax, has_nulls);
84  return chunk_metadata;
85  }

◆ readMetadata()

template<typename T, typename V>
void FixedLengthEncoder< T, V >::readMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 136 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, FixedLengthEncoder< T, V >::has_nulls, and Encoder::num_elems_.

136  {
137  // assumes pointer is already in right place
138  fread((int8_t*)&num_elems_, sizeof(size_t), 1, f);
139  fread((int8_t*)&dataMin, 1, sizeof(T), f);
140  fread((int8_t*)&dataMax, 1, sizeof(T), f);
141  fread((int8_t*)&has_nulls, 1, sizeof(bool), f);
142  }
size_t num_elems_
Definition: Encoder.h:179

◆ reduceStats()

template<typename T, typename V>
void FixedLengthEncoder< T, V >::reduceStats ( const Encoder that)
inlineoverridevirtual

Implements Encoder.

Definition at line 110 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, and FixedLengthEncoder< T, V >::has_nulls.

110  {
111  const auto that_typed = static_cast<const FixedLengthEncoder<T, V>&>(that);
112  if (that_typed.has_nulls) {
113  has_nulls = true;
114  }
115  dataMin = std::min(dataMin, that_typed.dataMin);
116  dataMax = std::max(dataMax, that_typed.dataMax);
117  }

◆ resetChunkStats()

template<typename T, typename V>
bool FixedLengthEncoder< T, V >::resetChunkStats ( const ChunkStats )
inlineoverridevirtual

: Reset chunk level stats (min, max, nulls) using new values from the argument.

Returns
: True if an update occurred and the chunk needs to be flushed. False otherwise. Default false if metadata update is unsupported. Only reset chunk stats if the incoming stats differ from the current stats.

Reimplemented from Encoder.

Definition at line 144 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, ChunkStats::has_nulls, FixedLengthEncoder< T, V >::has_nulls, ChunkStats::max, and ChunkStats::min.

144  {
145  const auto new_min = DatumFetcher::getDatumVal<T>(stats.min);
146  const auto new_max = DatumFetcher::getDatumVal<T>(stats.max);
147 
148  if (dataMin == new_min && dataMax == new_max && has_nulls == stats.has_nulls) {
149  return false;
150  }
151 
152  dataMin = new_min;
153  dataMax = new_max;
154  has_nulls = stats.has_nulls;
155  return true;
156  }

◆ updateStats() [1/2]

template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStats ( const int64_t  val,
const bool  is_null 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 88 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, and FixedLengthEncoder< T, V >::has_nulls.

88  {
89  if (is_null) {
90  has_nulls = true;
91  } else {
92  const auto data = static_cast<T>(val);
93  dataMin = std::min(dataMin, data);
94  dataMax = std::max(dataMax, data);
95  }
96  }
bool is_null(const T &v, const SQLTypeInfo &t)

◆ updateStats() [2/2]

template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStats ( const double  val,
const bool  is_null 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 99 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, and FixedLengthEncoder< T, V >::has_nulls.

99  {
100  if (is_null) {
101  has_nulls = true;
102  } else {
103  const auto data = static_cast<T>(val);
104  dataMin = std::min(dataMin, data);
105  dataMax = std::max(dataMax, data);
106  }
107  }
bool is_null(const T &v, const SQLTypeInfo &t)

◆ writeMetadata()

template<typename T, typename V>
void FixedLengthEncoder< T, V >::writeMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 128 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, FixedLengthEncoder< T, V >::has_nulls, and Encoder::num_elems_.

128  {
129  // assumes pointer is already in right place
130  fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
131  fwrite((int8_t*)&dataMin, sizeof(T), 1, f);
132  fwrite((int8_t*)&dataMax, sizeof(T), 1, f);
133  fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
134  }
size_t num_elems_
Definition: Encoder.h:179

Member Data Documentation

◆ dataMax

◆ dataMin

◆ has_nulls


The documentation for this class was generated from the following file: