OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Encoder Class Referenceabstract

#include <Encoder.h>

+ Inheritance diagram for Encoder:
+ Collaboration diagram for Encoder:

Public Member Functions

 Encoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~Encoder ()
 
virtual std::shared_ptr
< ChunkMetadata
appendData (int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1)=0
 
virtual void getMetadata (const std::shared_ptr< ChunkMetadata > &chunkMetadata)
 
virtual std::shared_ptr
< ChunkMetadata
getMetadata (const SQLTypeInfo &ti)=0
 
virtual void updateStats (const int64_t val, const bool is_null)=0
 
virtual void updateStats (const double val, const bool is_null)=0
 
virtual void updateStats (const int8_t *const src_data, const size_t num_elements)=0
 
virtual void updateStatsEncoded (const int8_t *const dst_data, const size_t num_elements)
 
virtual void updateStats (const std::vector< std::string > *const src_data, const size_t start_idx, const size_t num_elements)=0
 
virtual void updateStats (const std::vector< ArrayDatum > *const src_data, const size_t start_idx, const size_t num_elements)=0
 
virtual void reduceStats (const Encoder &)=0
 
virtual void copyMetadata (const Encoder *copyFromEncoder)=0
 
virtual void writeMetadata (FILE *f)=0
 
virtual void readMetadata (FILE *f)=0
 
virtual bool resetChunkStats (const ChunkStats &)
 : Reset chunk level stats (min, max, nulls) using new values from the argument. More...
 
size_t getNumElems () const
 
void setNumElems (const size_t num_elems)
 

Static Public Member Functions

static EncoderCreate (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
 

Protected Attributes

size_t num_elems_
 
Data_Namespace::AbstractBufferbuffer_
 
DecimalOverflowValidator decimal_overflow_validator_
 
DateDaysOverflowValidator date_days_overflow_validator_
 

Detailed Description

Definition at line 146 of file Encoder.h.

Constructor & Destructor Documentation

Encoder::Encoder ( Data_Namespace::AbstractBuffer buffer)

Definition at line 221 of file Encoder.cpp.

222  : num_elems_(0)
223  , buffer_(buffer)
224  , decimal_overflow_validator_(buffer ? buffer->getSqlType() : SQLTypeInfo())
225  , date_days_overflow_validator_(buffer ? buffer->getSqlType() : SQLTypeInfo()){};
size_t num_elems_
Definition: Encoder.h:229
DecimalOverflowValidator decimal_overflow_validator_
Definition: Encoder.h:233
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:231
DateDaysOverflowValidator date_days_overflow_validator_
Definition: Encoder.h:234
SQLTypeInfo getSqlType() const
virtual Encoder::~Encoder ( )
inlinevirtual

Definition at line 151 of file Encoder.h.

151 {}

Member Function Documentation

virtual std::shared_ptr<ChunkMetadata> Encoder::appendData ( int8_t *&  src_data,
const size_t  num_elems_to_append,
const SQLTypeInfo ti,
const bool  replicating = false,
const int64_t  offset = -1 
)
pure virtual

Append data to the chunk buffer backing this encoder.

Parameters
src_dataSource data for the append
num_elems_to_appendNumber of elements to append
tiSQL Type Info for the column TODO(adb): used?
replicatingPass one value and fill the chunk with it
offsetWrite data starting at a given offset. Default is -1 which indicates an append, an offset of 0 rewrites the chunk up to num_elems_to_append.

Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, StringNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, and DateDaysEncoder< T, V >.

Referenced by foreign_storage::ParquetGeospatialEncoder::appendBaseAndRenderGroupDataAndUpdateMetadata(), and Chunk_NS::Chunk::appendData().

+ Here is the caller graph for this function:

virtual void Encoder::copyMetadata ( const Encoder copyFromEncoder)
pure virtual
Encoder * Encoder::Create ( Data_Namespace::AbstractBuffer buffer,
const SQLTypeInfo  sqlType 
)
static

Definition at line 26 of file Encoder.cpp.

References CHECK, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), IS_STRING, SQLTypeInfo::is_string(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DATE_IN_DAYS, kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

Referenced by Data_Namespace::AbstractBuffer::initEncoder(), and synthesize_metadata().

27  {
28  switch (sqlType.get_compression()) {
29  case kENCODING_NONE: {
30  switch (sqlType.get_type()) {
31  case kBOOLEAN: {
32  return new NoneEncoder<int8_t>(buffer);
33  break;
34  }
35  case kTINYINT: {
36  return new NoneEncoder<int8_t>(buffer);
37  break;
38  }
39  case kSMALLINT: {
40  return new NoneEncoder<int16_t>(buffer);
41  break;
42  }
43  case kINT: {
44  return new NoneEncoder<int32_t>(buffer);
45  break;
46  }
47  case kBIGINT:
48  case kNUMERIC:
49  case kDECIMAL: {
50  return new NoneEncoder<int64_t>(buffer);
51  break;
52  }
53  case kFLOAT: {
54  return new NoneEncoder<float>(buffer);
55  break;
56  }
57  case kDOUBLE: {
58  return new NoneEncoder<double>(buffer);
59  break;
60  }
61  case kTEXT:
62  case kVARCHAR:
63  case kCHAR:
64  return new StringNoneEncoder(buffer);
65  case kARRAY: {
66  if (sqlType.get_size() > 0) {
67  return new FixedLengthArrayNoneEncoder(buffer, sqlType.get_size());
68  }
69  return new ArrayNoneEncoder(buffer);
70  }
71  case kTIME:
72  case kTIMESTAMP:
73  case kDATE:
74  return new NoneEncoder<int64_t>(buffer);
75  case kPOINT:
76  case kLINESTRING:
77  case kPOLYGON:
78  case kMULTIPOLYGON:
79  return new StringNoneEncoder(buffer);
80  default: {
81  return 0;
82  }
83  }
84  break;
85  }
87  switch (sqlType.get_type()) {
88  case kDATE:
89  switch (sqlType.get_comp_param()) {
90  case 0:
91  case 32:
92  return new DateDaysEncoder<int64_t, int32_t>(buffer);
93  break;
94  case 16:
95  return new DateDaysEncoder<int64_t, int16_t>(buffer);
96  break;
97  default:
98  return 0;
99  break;
100  }
101  break;
102  default:
103  return 0;
104  break;
105  }
106  }
107  case kENCODING_FIXED: {
108  switch (sqlType.get_type()) {
109  case kSMALLINT: {
110  switch (sqlType.get_comp_param()) {
111  case 8:
112  return new FixedLengthEncoder<int16_t, int8_t>(buffer);
113  break;
114  case 16:
115  return new NoneEncoder<int16_t>(buffer);
116  break;
117  default:
118  return 0;
119  break;
120  }
121  break;
122  }
123  case kINT: {
124  switch (sqlType.get_comp_param()) {
125  case 8:
126  return new FixedLengthEncoder<int32_t, int8_t>(buffer);
127  break;
128  case 16:
129  return new FixedLengthEncoder<int32_t, int16_t>(buffer);
130  break;
131  case 32:
132  return new NoneEncoder<int32_t>(buffer);
133  break;
134  default:
135  return 0;
136  break;
137  }
138  break;
139  }
140  case kBIGINT:
141  case kNUMERIC:
142  case kDECIMAL: {
143  switch (sqlType.get_comp_param()) {
144  case 8:
145  return new FixedLengthEncoder<int64_t, int8_t>(buffer);
146  break;
147  case 16:
148  return new FixedLengthEncoder<int64_t, int16_t>(buffer);
149  break;
150  case 32:
151  return new FixedLengthEncoder<int64_t, int32_t>(buffer);
152  break;
153  case 64:
154  return new NoneEncoder<int64_t>(buffer);
155  break;
156  default:
157  return 0;
158  break;
159  }
160  break;
161  }
162  case kTIME:
163  case kTIMESTAMP:
164  case kDATE:
165  return new FixedLengthEncoder<int64_t, int32_t>(buffer);
166  break;
167  default: {
168  return 0;
169  break;
170  }
171  } // switch (sqlType)
172  break;
173  } // Case: kENCODING_FIXED
174  case kENCODING_DICT: {
175  if (sqlType.get_type() == kARRAY) {
176  CHECK(IS_STRING(sqlType.get_subtype()));
177  if (sqlType.get_size() > 0) {
178  return new FixedLengthArrayNoneEncoder(buffer, sqlType.get_size());
179  }
180  return new ArrayNoneEncoder(buffer);
181  } else {
182  CHECK(sqlType.is_string());
183  switch (sqlType.get_size()) {
184  case 1:
185  return new NoneEncoder<uint8_t>(buffer);
186  break;
187  case 2:
188  return new NoneEncoder<uint16_t>(buffer);
189  break;
190  case 4:
191  return new NoneEncoder<int32_t>(buffer);
192  break;
193  default:
194  CHECK(false);
195  break;
196  }
197  }
198  break;
199  }
200  case kENCODING_GEOINT: {
201  switch (sqlType.get_type()) {
202  case kPOINT:
203  case kLINESTRING:
204  case kPOLYGON:
205  case kMULTIPOLYGON:
206  return new StringNoneEncoder(buffer);
207  default: {
208  return 0;
209  }
210  }
211  break;
212  }
213  default: {
214  return 0;
215  break;
216  }
217  } // switch (encodingType)
218  return 0;
219 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:331
HOST DEVICE int get_size() const
Definition: sqltypes.h:340
Definition: sqltypes.h:51
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:330
Definition: sqltypes.h:54
Definition: sqltypes.h:55
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:338
Definition: sqltypes.h:43
#define IS_STRING(T)
Definition: sqltypes.h:244
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:339
#define CHECK(condition)
Definition: Logger.h:197
Definition: sqltypes.h:47
bool is_string() const
Definition: sqltypes.h:487

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Encoder::getMetadata ( const std::shared_ptr< ChunkMetadata > &  chunkMetadata)
virtual

Reimplemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

Definition at line 227 of file Encoder.cpp.

References buffer_, Data_Namespace::AbstractBuffer::getSqlType(), num_elems_, and Data_Namespace::AbstractBuffer::size().

Referenced by StringNoneEncoder::getMetadata(), DateDaysEncoder< T, V >::getMetadata(), FixedLengthEncoder< T, V >::getMetadata(), NoneEncoder< T >::getMetadata(), FixedLengthArrayNoneEncoder::getMetadata(), and ArrayNoneEncoder::getMetadata().

227  {
228  chunkMetadata->sqlType = buffer_->getSqlType();
229  chunkMetadata->numBytes = buffer_->size();
230  chunkMetadata->numElements = num_elems_;
231 }
size_t num_elems_
Definition: Encoder.h:229
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:231
SQLTypeInfo getSqlType() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

virtual std::shared_ptr<ChunkMetadata> Encoder::getMetadata ( const SQLTypeInfo ti)
pure virtual
size_t Encoder::getNumElems ( ) const
inline

Definition at line 225 of file Encoder.h.

References num_elems_.

Referenced by StringNoneEncoder::copyMetadata(), DateDaysEncoder< T, V >::copyMetadata(), FixedLengthArrayNoneEncoder::copyMetadata(), FixedLengthEncoder< T, V >::copyMetadata(), NoneEncoder< T >::copyMetadata(), and ArrayNoneEncoder::copyMetadata().

225 { return num_elems_; }
size_t num_elems_
Definition: Encoder.h:229

+ Here is the caller graph for this function:

virtual void Encoder::readMetadata ( FILE *  f)
pure virtual
virtual void Encoder::reduceStats ( const Encoder )
pure virtual
virtual bool Encoder::resetChunkStats ( const ChunkStats )
inlinevirtual

: Reset chunk level stats (min, max, nulls) using new values from the argument.

Returns
: True if an update occurred and the chunk needs to be flushed. False otherwise. Default false if metadata update is unsupported. Only reset chunk stats if the incoming stats differ from the current stats.

Reimplemented in FixedLengthEncoder< T, V >, NoneEncoder< T >, and DateDaysEncoder< T, V >.

Definition at line 223 of file Encoder.h.

Referenced by foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::reduce_metadata(), and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer().

223 { return false; }

+ Here is the caller graph for this function:

void Encoder::setNumElems ( const size_t  num_elems)
inline

Definition at line 226 of file Encoder.h.

References num_elems_.

Referenced by foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer().

226 { num_elems_ = num_elems; }
size_t num_elems_
Definition: Encoder.h:229
size_t num_elems

+ Here is the caller graph for this function:

virtual void Encoder::updateStats ( const int64_t  val,
const bool  is_null 
)
pure virtual

Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

Referenced by foreign_storage::update_stats().

+ Here is the caller graph for this function:

virtual void Encoder::updateStats ( const double  val,
const bool  is_null 
)
pure virtual
virtual void Encoder::updateStats ( const int8_t *const  src_data,
const size_t  num_elements 
)
pure virtual

Update statistics for data without appending.

Parameters
src_data- the data with which to update statistics
num_elements- the number of elements to scan in the data

Implemented in ArrayNoneEncoder, FixedLengthArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, DateDaysEncoder< T, V >, and StringNoneEncoder.

virtual void Encoder::updateStats ( const std::vector< std::string > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
pure virtual

Update statistics for string data without appending.

Parameters
src_data- the string data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the string data

Implemented in ArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, FixedLengthArrayNoneEncoder, DateDaysEncoder< T, V >, and StringNoneEncoder.

virtual void Encoder::updateStats ( const std::vector< ArrayDatum > *const  src_data,
const size_t  start_idx,
const size_t  num_elements 
)
pure virtual

Update statistics for array data without appending.

Parameters
src_data- the array data with which to update statistics
start_idx- the offset into src_data to start the update
num_elements- the number of elements to scan in the array data

Implemented in ArrayNoneEncoder, NoneEncoder< T >, FixedLengthEncoder< T, V >, FixedLengthArrayNoneEncoder, DateDaysEncoder< T, V >, and StringNoneEncoder.

virtual void Encoder::updateStatsEncoded ( const int8_t *const  dst_data,
const size_t  num_elements 
)
inlinevirtual

Update statistics for encoded data without appending.

Parameters
dst_data- the data with which to update statistics
num_elements- the number of elements to scan in the data

Reimplemented in NoneEncoder< T >, and FixedLengthEncoder< T, V >.

Definition at line 185 of file Encoder.h.

References UNREACHABLE.

186  {
187  UNREACHABLE();
188  }
#define UNREACHABLE()
Definition: Logger.h:241
virtual void Encoder::writeMetadata ( FILE *  f)
pure virtual

Member Data Documentation

DateDaysOverflowValidator Encoder::date_days_overflow_validator_
protected

Definition at line 234 of file Encoder.h.

Referenced by DateDaysEncoder< T, V >::encodeDataAndUpdateStats().


The documentation for this class was generated from the following files: