OmniSciDB  ba1bac9284
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
FixedLengthEncoder< T, V > Class Template Reference

#include <FixedLengthEncoder.h>

+ Inheritance diagram for FixedLengthEncoder< T, V >:
+ Collaboration diagram for FixedLengthEncoder< T, V >:

Public Member Functions

 FixedLengthEncoder (Data_Namespace::AbstractBuffer *buffer)
 
std::shared_ptr< ChunkMetadataappendData (int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1) override
 
void getMetadata (const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
 
std::shared_ptr< ChunkMetadatagetMetadata (const SQLTypeInfo &ti) override
 
void updateStats (const int64_t val, const bool is_null) override
 
void updateStats (const double val, const bool is_null) override
 
void updateStats (const int8_t *const src_data, const size_t num_elements) override
 
void updateStatsEncoded (const int8_t *const dst_data, const size_t num_elements) override
 
void updateStats (const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements) override
 
void updateStats (const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements) override
 
void reduceStats (const Encoder &that) override
 
void copyMetadata (const Encoder *copyFromEncoder) override
 
void writeMetadata (FILE *f) override
 
void readMetadata (FILE *f) override
 
bool resetChunkStats (const ChunkStats &stats) override
 : Reset chunk level stats (min, max, nulls) using new values from the argument. More...
 
void resetChunkStats () override
 
- Public Member Functions inherited from Encoder
 Encoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~Encoder ()
 
size_t getNumElems () const
 
void setNumElems (const size_t num_elems)
 

Public Attributes

dataMin
 
dataMax
 
bool has_nulls
 

Private Member Functions

encodeDataAndUpdateStats (const T &unencoded_data)
 

Additional Inherited Members

- Static Public Member Functions inherited from Encoder
static EncoderCreate (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
 
- Protected Attributes inherited from Encoder
size_t num_elems_
 
Data_Namespace::AbstractBufferbuffer_
 
DecimalOverflowValidator decimal_overflow_validator_
 
DateDaysOverflowValidator date_days_overflow_validator_
 

Detailed Description

template<typename T, typename V>
class FixedLengthEncoder< T, V >

Definition at line 33 of file FixedLengthEncoder.h.

Constructor & Destructor Documentation

template<typename T, typename V>
FixedLengthEncoder< T, V >::FixedLengthEncoder ( Data_Namespace::AbstractBuffer buffer)
inline

Definition at line 35 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::resetChunkStats().

35  : Encoder(buffer) {
37  }
void resetChunkStats() override
Encoder(Data_Namespace::AbstractBuffer *buffer)
Definition: Encoder.cpp:221

+ Here is the call graph for this function:

Member Function Documentation

template<typename T, typename V>
std::shared_ptr<ChunkMetadata> FixedLengthEncoder< T, V >::appendData ( int8_t *&  src_data,
const size_t  num_elems_to_append,
const SQLTypeInfo ti,
const bool  replicating = false,
const int64_t  offset = -1 
)
inlineoverridevirtual

Append data to the chunk buffer backing this encoder.

Parameters
src_dataSource data for the append
num_elems_to_appendNumber of elements to append
tiSQL Type Info for the column TODO(adb): used?
replicatingPass one value and fill the chunk with it
offsetWrite data starting at a given offset. Default is -1 which indicates an append, an offset of 0 rewrites the chunk up to num_elems_to_append.

Implements Encoder.

Definition at line 39 of file FixedLengthEncoder.h.

References Data_Namespace::AbstractBuffer::append(), Encoder::buffer_, CHECK, CHECK_GE, FixedLengthEncoder< T, V >::encodeDataAndUpdateStats(), FixedLengthEncoder< T, V >::getMetadata(), i, Encoder::num_elems_, FixedLengthEncoder< T, V >::resetChunkStats(), omnisci.dtypes::T, and Data_Namespace::AbstractBuffer::write().

43  {
44  if (offset == 0 &&
45  num_elems_to_append >=
46  num_elems_) { // we're rewriting entire buffer so fully recompute metadata
48  }
49 
50  T* unencoded_data = reinterpret_cast<T*>(src_data);
51  auto encoded_data = std::make_unique<V[]>(num_elems_to_append);
52  for (size_t i = 0; i < num_elems_to_append; ++i) {
53  size_t ri = replicating ? 0 : i;
54  encoded_data.get()[i] = encodeDataAndUpdateStats(unencoded_data[ri]);
55  }
56 
57  // assume always CPU_BUFFER?
58  if (offset == -1) {
59  num_elems_ += num_elems_to_append;
60  buffer_->append(reinterpret_cast<int8_t*>(encoded_data.get()),
61  num_elems_to_append * sizeof(V));
62  if (!replicating) {
63  src_data += num_elems_to_append * sizeof(T);
64  }
65  } else {
66  num_elems_ = offset + num_elems_to_append;
67  CHECK(!replicating);
68  CHECK_GE(offset, 0);
69  buffer_->write(reinterpret_cast<int8_t*>(encoded_data.get()),
70  num_elems_to_append * sizeof(V),
71  static_cast<size_t>(offset));
72  }
73  auto chunk_metadata = std::make_shared<ChunkMetadata>();
74  getMetadata(chunk_metadata);
75  return chunk_metadata;
76  }
size_t num_elems_
Definition: Encoder.h:237
#define CHECK_GE(x, y)
Definition: Logger.h:219
void resetChunkStats() override
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:239
V encodeDataAndUpdateStats(const T &unencoded_data)
void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
virtual void write(int8_t *src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int src_device_id=-1)=0
virtual void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
#define CHECK(condition)
Definition: Logger.h:206

+ Here is the call graph for this function:

template<typename T, typename V>
void FixedLengthEncoder< T, V >::copyMetadata ( const Encoder copyFromEncoder)
inlineoverridevirtual

Implements Encoder.

Definition at line 170 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, Encoder::getNumElems(), FixedLengthEncoder< T, V >::has_nulls, and Encoder::num_elems_.

170  {
171  num_elems_ = copyFromEncoder->getNumElems();
172  auto castedEncoder =
173  reinterpret_cast<const FixedLengthEncoder<T, V>*>(copyFromEncoder);
174  dataMin = castedEncoder->dataMin;
175  dataMax = castedEncoder->dataMax;
176  has_nulls = castedEncoder->has_nulls;
177  }
size_t num_elems_
Definition: Encoder.h:237
size_t getNumElems() const
Definition: Encoder.h:233

+ Here is the call graph for this function:

template<typename T, typename V>
V FixedLengthEncoder< T, V >::encodeDataAndUpdateStats ( const T &  unencoded_data)
inlineprivate

Definition at line 220 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, Encoder::decimal_overflow_validator_, logger::ERROR, FixedLengthEncoder< T, V >::has_nulls, LOG, omnisci.dtypes::T, to_string(), and DecimalOverflowValidator::validate().

Referenced by FixedLengthEncoder< T, V >::appendData(), and FixedLengthEncoder< T, V >::updateStats().

220  {
221  V encoded_data = static_cast<V>(unencoded_data);
222  if (unencoded_data != encoded_data) {
223  decimal_overflow_validator_.validate(unencoded_data);
224  LOG(ERROR) << "Fixed encoding failed, Unencoded: " +
225  std::to_string(unencoded_data) +
226  " encoded: " + std::to_string(encoded_data);
227  } else {
228  T data = unencoded_data;
229  if (data == std::numeric_limits<V>::min()) {
230  has_nulls = true;
231  } else {
233  dataMin = std::min(dataMin, data);
234  dataMax = std::max(dataMax, data);
235  }
236  }
237  return encoded_data;
238  }
DecimalOverflowValidator decimal_overflow_validator_
Definition: Encoder.h:241
#define LOG(tag)
Definition: Logger.h:200
std::string to_string(char const *&&v)
void validate(T value)
Definition: Encoder.h:54

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T, typename V>
void FixedLengthEncoder< T, V >::getMetadata ( const std::shared_ptr< ChunkMetadata > &  chunkMetadata)
inlineoverridevirtual

Reimplemented from Encoder.

Definition at line 78 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, Encoder::getMetadata(), and FixedLengthEncoder< T, V >::has_nulls.

Referenced by FixedLengthEncoder< T, V >::appendData().

78  {
79  Encoder::getMetadata(chunkMetadata); // call on parent class
80  chunkMetadata->fillChunkStats(dataMin, dataMax, has_nulls);
81  }
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:227

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename T, typename V>
std::shared_ptr<ChunkMetadata> FixedLengthEncoder< T, V >::getMetadata ( const SQLTypeInfo ti)
inlineoverridevirtual

Implements Encoder.

Definition at line 84 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, and FixedLengthEncoder< T, V >::has_nulls.

84  {
85  auto chunk_metadata = std::make_shared<ChunkMetadata>(ti, 0, 0, ChunkStats{});
86  chunk_metadata->fillChunkStats(dataMin, dataMax, has_nulls);
87  return chunk_metadata;
88  }
template<typename T, typename V>
void FixedLengthEncoder< T, V >::readMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 187 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, FixedLengthEncoder< T, V >::has_nulls, Encoder::num_elems_, and omnisci.dtypes::T.

187  {
188  // assumes pointer is already in right place
189  fread((int8_t*)&num_elems_, sizeof(size_t), 1, f);
190  fread((int8_t*)&dataMin, 1, sizeof(T), f);
191  fread((int8_t*)&dataMax, 1, sizeof(T), f);
192  fread((int8_t*)&has_nulls, 1, sizeof(bool), f);
193  }
size_t num_elems_
Definition: Encoder.h:237
char * f
template<typename T, typename V>
void FixedLengthEncoder< T, V >::reduceStats ( const Encoder that)
inlineoverridevirtual

Implements Encoder.

Definition at line 161 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, and FixedLengthEncoder< T, V >::has_nulls.

161  {
162  const auto that_typed = static_cast<const FixedLengthEncoder<T, V>&>(that);
163  if (that_typed.has_nulls) {
164  has_nulls = true;
165  }
166  dataMin = std::min(dataMin, that_typed.dataMin);
167  dataMax = std::max(dataMax, that_typed.dataMax);
168  }
template<typename T, typename V>
bool FixedLengthEncoder< T, V >::resetChunkStats ( const ChunkStats )
inlineoverridevirtual

: Reset chunk level stats (min, max, nulls) using new values from the argument.

Returns
: True if an update occurred and the chunk needs to be flushed. False otherwise. Default false if metadata update is unsupported. Only reset chunk stats if the incoming stats differ from the current stats.

Reimplemented from Encoder.

Definition at line 195 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, ChunkStats::has_nulls, FixedLengthEncoder< T, V >::has_nulls, ChunkStats::max, and ChunkStats::min.

195  {
196  const auto new_min = DatumFetcher::getDatumVal<T>(stats.min);
197  const auto new_max = DatumFetcher::getDatumVal<T>(stats.max);
198 
199  if (dataMin == new_min && dataMax == new_max && has_nulls == stats.has_nulls) {
200  return false;
201  }
202 
203  dataMin = new_min;
204  dataMax = new_max;
205  has_nulls = stats.has_nulls;
206  return true;
207  }
template<typename T, typename V>
void FixedLengthEncoder< T, V >::resetChunkStats ( )
inlineoverridevirtual

Resets chunk metadata stats to their default values.

Implements Encoder.

Definition at line 209 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, and FixedLengthEncoder< T, V >::has_nulls.

Referenced by FixedLengthEncoder< T, V >::appendData(), and FixedLengthEncoder< T, V >::FixedLengthEncoder().

209  {
210  dataMin = std::numeric_limits<T>::max();
211  dataMax = std::numeric_limits<T>::lowest();
212  has_nulls = false;
213  }

+ Here is the caller graph for this function:

template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStats ( const int64_t  val,
const bool  is_null 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 91 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, FixedLengthEncoder< T, V >::has_nulls, and omnisci.dtypes::T.

91  {
92  if (is_null) {
93  has_nulls = true;
94  } else {
95  const auto data = static_cast<T>(val);
96  dataMin = std::min(dataMin, data);
97  dataMax = std::max(dataMax, data);
98  }
99  }
CONSTEXPR DEVICE bool is_null(const T &value)
template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStats ( const double  val,
const bool  is_null 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 102 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, FixedLengthEncoder< T, V >::has_nulls, and omnisci.dtypes::T.

102  {
103  if (is_null) {
104  has_nulls = true;
105  } else {
106  const auto data = static_cast<T>(val);
107  dataMin = std::min(dataMin, data);
108  dataMax = std::max(dataMax, data);
109  }
110  }
CONSTEXPR DEVICE bool is_null(const T &value)
template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStats ( const int8_t *const  src_data,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for data without appending.

Parameters
src_data- the data with which to update statistics
num_elements- the number of elements to scan in the data

Implements Encoder.

Definition at line 112 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::encodeDataAndUpdateStats(), i, and omnisci.dtypes::T.

112  {
113  const T* unencoded_data = reinterpret_cast<const T*>(src_data);
114  for (size_t i = 0; i < num_elements; ++i) {
115  encodeDataAndUpdateStats(unencoded_data[i]);
116  }
117  }
V encodeDataAndUpdateStats(const T &unencoded_data)

+ Here is the call graph for this function:

template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStats ( const std::vector< std::string > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for string data without appending.

Parameters
src_data- the string data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the string data

Implements Encoder.

Definition at line 148 of file FixedLengthEncoder.h.

References UNREACHABLE.

150  {
151  UNREACHABLE();
152  }
#define UNREACHABLE()
Definition: Logger.h:250
template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStats ( const std::vector< ArrayDatum > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for array data without appending.

Parameters
src_data- the array data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the array data

Implements Encoder.

Definition at line 154 of file FixedLengthEncoder.h.

References UNREACHABLE.

156  {
157  UNREACHABLE();
158  }
#define UNREACHABLE()
Definition: Logger.h:250
template<typename T, typename V>
void FixedLengthEncoder< T, V >::updateStatsEncoded ( const int8_t *const  dst_data,
const size_t  num_elements 
)
inlineoverridevirtual

Update statistics for encoded data without appending.

Parameters
dst_data- the data with which to update statistics
num_elements- the number of elements to scan in the data

Reimplemented from Encoder.

Definition at line 119 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, Encoder::decimal_overflow_validator_, FixedLengthEncoder< T, V >::has_nulls, i, logger::init(), and DecimalOverflowValidator::validate().

120  {
121  const V* data = reinterpret_cast<const V*>(dst_data);
122 
123  std::tie(dataMin, dataMax, has_nulls) = tbb::parallel_reduce(
124  tbb::blocked_range(size_t(0), num_elements),
125  std::tuple(static_cast<V>(dataMin), static_cast<V>(dataMax), has_nulls),
126  [&](const auto& range, auto init) {
127  auto [min, max, nulls] = init;
128  for (size_t i = range.begin(); i < range.end(); i++) {
129  if (data[i] != std::numeric_limits<V>::min()) {
131  min = std::min(min, data[i]);
132  max = std::max(max, data[i]);
133  } else {
134  nulls = true;
135  }
136  }
137  return std::tuple(min, max, nulls);
138  },
139  [&](auto lhs, auto rhs) {
140  const auto [lhs_min, lhs_max, lhs_nulls] = lhs;
141  const auto [rhs_min, rhs_max, rhs_nulls] = rhs;
142  return std::tuple(std::min(lhs_min, rhs_min),
143  std::max(lhs_max, rhs_max),
144  lhs_nulls || rhs_nulls);
145  });
146  }
DecimalOverflowValidator decimal_overflow_validator_
Definition: Encoder.h:241
void init(LogOptions const &log_opts)
Definition: Logger.cpp:280
void validate(T value)
Definition: Encoder.h:54

+ Here is the call graph for this function:

template<typename T, typename V>
void FixedLengthEncoder< T, V >::writeMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 179 of file FixedLengthEncoder.h.

References FixedLengthEncoder< T, V >::dataMax, FixedLengthEncoder< T, V >::dataMin, FixedLengthEncoder< T, V >::has_nulls, Encoder::num_elems_, and omnisci.dtypes::T.

179  {
180  // assumes pointer is already in right place
181  fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
182  fwrite((int8_t*)&dataMin, sizeof(T), 1, f);
183  fwrite((int8_t*)&dataMax, sizeof(T), 1, f);
184  fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
185  }
size_t num_elems_
Definition: Encoder.h:237
char * f

Member Data Documentation


The documentation for this class was generated from the following file: