OmniSciDB  8a228a1076
Encoder Class Referenceabstract

#include <Encoder.h>

+ Inheritance diagram for Encoder:
+ Collaboration diagram for Encoder:

Public Member Functions

 Encoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~Encoder ()
 
virtual std::shared_ptr< ChunkMetadataappendData (int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1)=0
 
virtual void getMetadata (const std::shared_ptr< ChunkMetadata > &chunkMetadata)
 
virtual std::shared_ptr< ChunkMetadatagetMetadata (const SQLTypeInfo &ti)=0
 
virtual void updateStats (const int64_t val, const bool is_null)=0
 
virtual void updateStats (const double val, const bool is_null)=0
 
virtual void updateStats (const int8_t *const src_data, const size_t num_elements)=0
 
virtual void updateStats (const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements)=0
 
virtual void updateStats (const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements)=0
 
virtual void reduceStats (const Encoder &)=0
 
virtual void copyMetadata (const Encoder *copyFromEncoder)=0
 
virtual void writeMetadata (FILE *f)=0
 
virtual void readMetadata (FILE *f)=0
 
virtual bool resetChunkStats (const ChunkStats &)
 : Reset chunk level stats (min, max, nulls) using new values from the argument. More...
 
size_t getNumElems () const
 
void setNumElems (const size_t num_elems)
 

Static Public Member Functions

static EncoderCreate (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
 

Protected Attributes

size_t num_elems_
 
Data_Namespace::AbstractBufferbuffer_
 
DecimalOverflowValidator decimal_overflow_validator_
 
DateDaysOverflowValidator date_days_overflow_validator_
 

Detailed Description

Definition at line 146 of file Encoder.h.

Constructor & Destructor Documentation

◆ Encoder()

Encoder::Encoder ( Data_Namespace::AbstractBuffer buffer)

Definition at line 221 of file Encoder.cpp.

222  : num_elems_(0)
223  , buffer_(buffer)
224  , decimal_overflow_validator_(buffer ? buffer->sql_type : SQLTypeInfo())
225  , date_days_overflow_validator_(buffer ? buffer->sql_type : SQLTypeInfo()){};
size_t num_elems_
Definition: Encoder.h:218
DecimalOverflowValidator decimal_overflow_validator_
Definition: Encoder.h:222
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:220
DateDaysOverflowValidator date_days_overflow_validator_
Definition: Encoder.h:223

◆ ~Encoder()

virtual Encoder::~Encoder ( )
inlinevirtual

Definition at line 151 of file Encoder.h.

References anonymous_namespace{TypedDataAccessors.h}::is_null().

151 {}
+ Here is the call graph for this function:

Member Function Documentation

◆ appendData()

virtual std::shared_ptr<ChunkMetadata> Encoder::appendData ( int8_t *&  src_data,
const size_t  num_elems_to_append,
const SQLTypeInfo ti,
const bool  replicating = false,
const int64_t  offset = -1 
)
pure virtual

Append data to the chunk buffer backing this encoder.

Parameters
src_dataSource data for the append
num_elems_to_appendNumber of elements to append
tiSQL Type Info for the column TODO(adb): used?
replicatingPass one value and fill the chunk with it
offsetWrite data starting at a given offset. Default is -1 which indicates an append, an offset of 0 rewrites the chunk up to num_elems_to_append.

Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, StringNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, and DateDaysEncoder< T, V >.

◆ copyMetadata()

virtual void Encoder::copyMetadata ( const Encoder copyFromEncoder)
pure virtual

◆ Create()

Encoder * Encoder::Create ( Data_Namespace::AbstractBuffer buffer,
const SQLTypeInfo  sqlType 
)
static

Definition at line 26 of file Encoder.cpp.

References CHECK, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), IS_STRING, SQLTypeInfo::is_string(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DATE_IN_DAYS, kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

Referenced by ArrowCsvForeignStorage::createDecimalColumn(), ArrowCsvForeignStorage::createDictionaryEncodedColumn(), Data_Namespace::AbstractBuffer::initEncoder(), ArrowCsvForeignStorage::registerTable(), and synthesize_metadata().

27  {
28  switch (sqlType.get_compression()) {
29  case kENCODING_NONE: {
30  switch (sqlType.get_type()) {
31  case kBOOLEAN: {
32  return new NoneEncoder<int8_t>(buffer);
33  break;
34  }
35  case kTINYINT: {
36  return new NoneEncoder<int8_t>(buffer);
37  break;
38  }
39  case kSMALLINT: {
40  return new NoneEncoder<int16_t>(buffer);
41  break;
42  }
43  case kINT: {
44  return new NoneEncoder<int32_t>(buffer);
45  break;
46  }
47  case kBIGINT:
48  case kNUMERIC:
49  case kDECIMAL: {
50  return new NoneEncoder<int64_t>(buffer);
51  break;
52  }
53  case kFLOAT: {
54  return new NoneEncoder<float>(buffer);
55  break;
56  }
57  case kDOUBLE: {
58  return new NoneEncoder<double>(buffer);
59  break;
60  }
61  case kTEXT:
62  case kVARCHAR:
63  case kCHAR:
64  return new StringNoneEncoder(buffer);
65  case kARRAY: {
66  if (sqlType.get_size() > 0) {
67  return new FixedLengthArrayNoneEncoder(buffer, sqlType.get_size());
68  }
69  return new ArrayNoneEncoder(buffer);
70  }
71  case kTIME:
72  case kTIMESTAMP:
73  case kDATE:
74  return new NoneEncoder<int64_t>(buffer);
75  case kPOINT:
76  case kLINESTRING:
77  case kPOLYGON:
78  case kMULTIPOLYGON:
79  return new StringNoneEncoder(buffer);
80  default: {
81  return 0;
82  }
83  }
84  break;
85  }
87  switch (sqlType.get_type()) {
88  case kDATE:
89  switch (sqlType.get_comp_param()) {
90  case 0:
91  case 32:
92  return new DateDaysEncoder<int64_t, int32_t>(buffer);
93  break;
94  case 16:
95  return new DateDaysEncoder<int64_t, int16_t>(buffer);
96  break;
97  default:
98  return 0;
99  break;
100  }
101  break;
102  default:
103  return 0;
104  break;
105  }
106  }
107  case kENCODING_FIXED: {
108  switch (sqlType.get_type()) {
109  case kSMALLINT: {
110  switch (sqlType.get_comp_param()) {
111  case 8:
112  return new FixedLengthEncoder<int16_t, int8_t>(buffer);
113  break;
114  case 16:
115  return new NoneEncoder<int16_t>(buffer);
116  break;
117  default:
118  return 0;
119  break;
120  }
121  break;
122  }
123  case kINT: {
124  switch (sqlType.get_comp_param()) {
125  case 8:
126  return new FixedLengthEncoder<int32_t, int8_t>(buffer);
127  break;
128  case 16:
129  return new FixedLengthEncoder<int32_t, int16_t>(buffer);
130  break;
131  case 32:
132  return new NoneEncoder<int32_t>(buffer);
133  break;
134  default:
135  return 0;
136  break;
137  }
138  break;
139  }
140  case kBIGINT:
141  case kNUMERIC:
142  case kDECIMAL: {
143  switch (sqlType.get_comp_param()) {
144  case 8:
145  return new FixedLengthEncoder<int64_t, int8_t>(buffer);
146  break;
147  case 16:
148  return new FixedLengthEncoder<int64_t, int16_t>(buffer);
149  break;
150  case 32:
151  return new FixedLengthEncoder<int64_t, int32_t>(buffer);
152  break;
153  case 64:
154  return new NoneEncoder<int64_t>(buffer);
155  break;
156  default:
157  return 0;
158  break;
159  }
160  break;
161  }
162  case kTIME:
163  case kTIMESTAMP:
164  case kDATE:
165  return new FixedLengthEncoder<int64_t, int32_t>(buffer);
166  break;
167  default: {
168  return 0;
169  break;
170  }
171  } // switch (sqlType)
172  break;
173  } // Case: kENCODING_FIXED
174  case kENCODING_DICT: {
175  if (sqlType.get_type() == kARRAY) {
176  CHECK(IS_STRING(sqlType.get_subtype()));
177  if (sqlType.get_size() > 0) {
178  return new FixedLengthArrayNoneEncoder(buffer, sqlType.get_size());
179  }
180  return new ArrayNoneEncoder(buffer);
181  } else {
182  CHECK(sqlType.is_string());
183  switch (sqlType.get_size()) {
184  case 1:
185  return new NoneEncoder<uint8_t>(buffer);
186  break;
187  case 2:
188  return new NoneEncoder<uint16_t>(buffer);
189  break;
190  case 4:
191  return new NoneEncoder<int32_t>(buffer);
192  break;
193  default:
194  CHECK(false);
195  break;
196  }
197  }
198  break;
199  }
200  case kENCODING_GEOINT: {
201  switch (sqlType.get_type()) {
202  case kPOINT:
203  case kLINESTRING:
204  case kPOLYGON:
205  case kMULTIPOLYGON:
206  return new StringNoneEncoder(buffer);
207  default: {
208  return 0;
209  }
210  }
211  break;
212  }
213  default: {
214  return 0;
215  break;
216  }
217  } // switch (encodingType)
218  return 0;
219 }
bool is_string() const
Definition: sqltypes.h:416
Definition: sqltypes.h:51
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:268
HOST DEVICE int get_size() const
Definition: sqltypes.h:269
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:267
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:260
Definition: sqltypes.h:54
Definition: sqltypes.h:55
Definition: sqltypes.h:43
#define IS_STRING(T)
Definition: sqltypes.h:173
#define CHECK(condition)
Definition: Logger.h:197
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:259
Definition: sqltypes.h:47
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadata() [1/2]

void Encoder::getMetadata ( const std::shared_ptr< ChunkMetadata > &  chunkMetadata)
virtual

Reimplemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

Definition at line 227 of file Encoder.cpp.

References buffer_, num_elems_, Data_Namespace::AbstractBuffer::size(), and Data_Namespace::AbstractBuffer::sql_type.

Referenced by StringNoneEncoder::getMetadata(), DateDaysEncoder< T, V >::getMetadata(), FixedLengthEncoder< T, V >::getMetadata(), NoneEncoder< T >::getMetadata(), FixedLengthArrayNoneEncoder::getMetadata(), and ArrayNoneEncoder::getMetadata().

227  {
228  chunkMetadata->sqlType = buffer_->sql_type;
229  chunkMetadata->numBytes = buffer_->size();
230  chunkMetadata->numElements = num_elems_;
231 }
size_t num_elems_
Definition: Encoder.h:218
virtual size_t size() const =0
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:220
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadata() [2/2]

virtual std::shared_ptr<ChunkMetadata> Encoder::getMetadata ( const SQLTypeInfo ti)
pure virtual

◆ getNumElems()

size_t Encoder::getNumElems ( ) const
inline

Definition at line 214 of file Encoder.h.

Referenced by StringNoneEncoder::copyMetadata(), DateDaysEncoder< T, V >::copyMetadata(), FixedLengthEncoder< T, V >::copyMetadata(), FixedLengthArrayNoneEncoder::copyMetadata(), NoneEncoder< T >::copyMetadata(), ArrayNoneEncoder::copyMetadata(), and foreign_storage::ParquetDataWrapper::updateStatsForEncoder().

214 { return num_elems_; }
size_t num_elems_
Definition: Encoder.h:218
+ Here is the caller graph for this function:

◆ readMetadata()

virtual void Encoder::readMetadata ( FILE *  f)
pure virtual

◆ reduceStats()

virtual void Encoder::reduceStats ( const Encoder )
pure virtual

◆ resetChunkStats()

virtual bool Encoder::resetChunkStats ( const ChunkStats )
inlinevirtual

: Reset chunk level stats (min, max, nulls) using new values from the argument.

Returns
: True if an update occurred and the chunk needs to be flushed. False otherwise. Default false if metadata update is unsupported. Only reset chunk stats if the incoming stats differ from the current stats.

Reimplemented in FixedLengthEncoder< T, V >, NoneEncoder< T >, and DateDaysEncoder< T, V >.

Definition at line 212 of file Encoder.h.

212 { return false; }

◆ setNumElems()

void Encoder::setNumElems ( const size_t  num_elems)
inline

Definition at line 215 of file Encoder.h.

Referenced by foreign_storage::ParquetDataWrapper::updateStatsForEncoder().

215 { num_elems_ = num_elems; }
size_t num_elems_
Definition: Encoder.h:218
size_t num_elems
+ Here is the caller graph for this function:

◆ updateStats() [1/5]

virtual void Encoder::updateStats ( const int64_t  val,
const bool  is_null 
)
pure virtual

Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

Referenced by foreign_storage::update_stats(), and foreign_storage::ParquetDataWrapper::updateStatsForEncoder().

+ Here is the caller graph for this function:

◆ updateStats() [2/5]

virtual void Encoder::updateStats ( const double  val,
const bool  is_null 
)
pure virtual

◆ updateStats() [3/5]

virtual void Encoder::updateStats ( const int8_t *const  src_data,
const size_t  num_elements 
)
pure virtual

Update statistics for data without appending.

Parameters
src_data- the data with which to update statistics
num_elements- the number of elements to scan in the data

Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

◆ updateStats() [4/5]

virtual void Encoder::updateStats ( const std::vector< std::string > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
pure virtual

Update statistics for string data without appending.

Parameters
src_data- the string data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the string data

Implemented in ArrayNoneEncoder, NoneEncoder< T >, FixedLengthArrayNoneEncoder, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

◆ updateStats() [5/5]

virtual void Encoder::updateStats ( const std::vector< ArrayDatum > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
pure virtual

Update statistics for array data without appending.

Parameters
src_data- the array data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the array data

Implemented in ArrayNoneEncoder, NoneEncoder< T >, FixedLengthArrayNoneEncoder, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

◆ writeMetadata()

virtual void Encoder::writeMetadata ( FILE *  f)
pure virtual

Member Data Documentation

◆ buffer_

◆ date_days_overflow_validator_

DateDaysOverflowValidator Encoder::date_days_overflow_validator_
protected

Definition at line 223 of file Encoder.h.

Referenced by DateDaysEncoder< T, V >::encodeDataAndUpdateStats().

◆ decimal_overflow_validator_

◆ num_elems_


The documentation for this class was generated from the following files: