OmniSciDB  2e3a973ef4
Public Member Functions | Protected Member Functions | Private Member Functions | Private Attributes | List of all members
foreign_storage::ParquetFixedLengthArrayEncoder Class Reference

#include <ParquetFixedLengthArrayEncoder.h>

+ Inheritance diagram for foreign_storage::ParquetFixedLengthArrayEncoder:
+ Collaboration diagram for foreign_storage::ParquetFixedLengthArrayEncoder:

Public Member Functions

 ParquetFixedLengthArrayEncoder (Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
 
- Public Member Functions inherited from foreign_storage::ParquetArrayEncoder
 ParquetArrayEncoder (Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
 
void appendData (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, const bool is_last_batch, int8_t *values) override
 
- Public Member Functions inherited from foreign_storage::ParquetEncoder
 ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~ParquetEncoder ()=default
 

Protected Member Functions

void processLastArray () override
 
- Protected Member Functions inherited from foreign_storage::ParquetArrayEncoder
virtual void appendArraysToBuffer ()
 
bool isLastArrayNull () const
 
size_t sizeOfLastArray () const
 
int8_t * resizeArrayDataBytes (const size_t additional_num_elements)
 

Private Member Functions

void appendNullFixedLengthArray ()
 
void setNullFixedLengthArraySentinel (int8_t *omnisci_data_bytes)
 
void appendNullArrayOrCheckArraySize ()
 

Private Attributes

const ColumnDescriptor column_desciptor_
 
size_t array_element_count_
 

Additional Inherited Members

- Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
size_t omnisci_data_type_byte_size_
 
std::shared_ptr< ParquetScalarEncoder > scalar_encoder_
 
std::vector< int8_t > data_buffer_bytes_
 
- Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer * buffer_
 
- Static Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
static const int16_t non_null_def_level = 3
 
static const int16_t item_null_def_level = 2
 
static const int16_t list_null_def_level = 0
 

Detailed Description

Definition at line 27 of file ParquetFixedLengthArrayEncoder.h.

Constructor & Destructor Documentation

◆ ParquetFixedLengthArrayEncoder()

foreign_storage::ParquetFixedLengthArrayEncoder::ParquetFixedLengthArrayEncoder ( Data_Namespace::AbstractBuffer *  data_buffer,
std::shared_ptr< ParquetScalarEncoder >  scalar_encoder,
const ColumnDescriptor *  column_desciptor 
)
inline

Definition at line 29 of file ParquetFixedLengthArrayEncoder.h.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfo::get_size(), and foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_.

32  : ParquetArrayEncoder(data_buffer, scalar_encoder, column_desciptor)
33  , column_desciptor_(*column_desciptor)
34  , array_element_count_(column_desciptor->columnType.get_size() /
35  omnisci_data_type_byte_size_) {
36  CHECK(column_desciptor->columnType.get_size() % omnisci_data_type_byte_size_ == 0);
37  }
foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_
size_t omnisci_data_type_byte_size_
Definition: ParquetArrayEncoder.h:81
SQLTypeInfo::get_size
HOST DEVICE int get_size() const
Definition: sqltypes.h:269
foreign_storage::ParquetArrayEncoder::ParquetArrayEncoder
ParquetArrayEncoder(Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
Definition: ParquetArrayEncoder.h:25
foreign_storage::ParquetFixedLengthArrayEncoder::column_desciptor_
const ColumnDescriptor column_desciptor_
Definition: ParquetFixedLengthArrayEncoder.h:131
CHECK
#define CHECK(condition)
Definition: Logger.h:197
foreign_storage::ParquetFixedLengthArrayEncoder::array_element_count_
size_t array_element_count_
Definition: ParquetFixedLengthArrayEncoder.h:132
ColumnDescriptor::columnType
SQLTypeInfo columnType
Definition: ColumnDescriptor.h:35
+ Here is the call graph for this function:

Member Function Documentation

◆ appendNullArrayOrCheckArraySize()

void foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize ( )
inlineprivate

Definition at line 111 of file ParquetFixedLengthArrayEncoder.h.

References appendNullFixedLengthArray(), array_element_count_, CHECK, column_desciptor_, ColumnDescriptor::columnName, foreign_storage::ParquetArrayEncoder::isLastArrayNull(), foreign_storage::ParquetArrayEncoder::sizeOfLastArray(), and to_string().

Referenced by processLastArray().

111  {
112  auto size_of_last_array = sizeOfLastArray();
113  if (!isLastArrayNull()) {
114  if (size_of_last_array != array_element_count_) {
115  throw std::runtime_error("Detected a row with " +
116  std::to_string(size_of_last_array) +
117  " elements being loaded into"
118  " OmniSci column '" +
119  column_desciptor_.columnName +
120  "' which has a fixed length array type,"
121  " expecting " +
122  std::to_string(array_element_count_) + " elements.");
123  }
124  } else {
125  // append a null array sentinel
126  CHECK(size_of_last_array == 0);
127  appendNullFixedLengthArray();
128  }
129  }
foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray
void appendNullFixedLengthArray()
Definition: ParquetFixedLengthArrayEncoder.h:43
to_string
std::string to_string(char const *&&v)
Definition: StringTransform.cpp:86
foreign_storage::ParquetArrayEncoder::isLastArrayNull
bool isLastArrayNull() const
Definition: ParquetArrayEncoder.h:70
foreign_storage::ParquetFixedLengthArrayEncoder::column_desciptor_
const ColumnDescriptor column_desciptor_
Definition: ParquetFixedLengthArrayEncoder.h:131
CHECK
#define CHECK(condition)
Definition: Logger.h:197
foreign_storage::ParquetFixedLengthArrayEncoder::array_element_count_
size_t array_element_count_
Definition: ParquetFixedLengthArrayEncoder.h:132
foreign_storage::ParquetArrayEncoder::sizeOfLastArray
size_t sizeOfLastArray() const
Definition: ParquetArrayEncoder.h:72
ColumnDescriptor::columnName
std::string columnName
Definition: ColumnDescriptor.h:33
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ appendNullFixedLengthArray()

void foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray ( )
inlineprivate

Definition at line 43 of file ParquetFixedLengthArrayEncoder.h.

References array_element_count_, foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_, foreign_storage::ParquetArrayEncoder::resizeArrayDataBytes(), foreign_storage::ParquetArrayEncoder::scalar_encoder_, and setNullFixedLengthArraySentinel().

Referenced by appendNullArrayOrCheckArraySize().

43  {
44  auto omnisci_data_ptr = resizeArrayDataBytes(array_element_count_);
45  setNullFixedLengthArraySentinel(omnisci_data_ptr);
46  for (size_t i = 1; i < array_element_count_; ++i) {
47  scalar_encoder_->setNull(omnisci_data_ptr + i * omnisci_data_type_byte_size_);
48  }
49  }
foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_
size_t omnisci_data_type_byte_size_
Definition: ParquetArrayEncoder.h:81
foreign_storage::ParquetArrayEncoder::resizeArrayDataBytes
int8_t * resizeArrayDataBytes(const size_t additional_num_elements)
Definition: ParquetArrayEncoder.h:74
foreign_storage::ParquetFixedLengthArrayEncoder::setNullFixedLengthArraySentinel
void setNullFixedLengthArraySentinel(int8_t *omnisci_data_bytes)
Definition: ParquetFixedLengthArrayEncoder.h:51
foreign_storage::ParquetFixedLengthArrayEncoder::array_element_count_
size_t array_element_count_
Definition: ParquetFixedLengthArrayEncoder.h:132
foreign_storage::ParquetArrayEncoder::scalar_encoder_
std::shared_ptr< ParquetScalarEncoder > scalar_encoder_
Definition: ParquetArrayEncoder.h:82
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ processLastArray()

void foreign_storage::ParquetFixedLengthArrayEncoder::processLastArray ( )
inlineoverrideprotectedvirtual

Implements foreign_storage::ParquetArrayEncoder.

Definition at line 40 of file ParquetFixedLengthArrayEncoder.h.

References appendNullArrayOrCheckArraySize().

40 { appendNullArrayOrCheckArraySize(); }
foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize
void appendNullArrayOrCheckArraySize()
Definition: ParquetFixedLengthArrayEncoder.h:111
+ Here is the call graph for this function:

◆ setNullFixedLengthArraySentinel()

void foreign_storage::ParquetFixedLengthArrayEncoder::setNullFixedLengthArraySentinel ( int8_t *  omnisci_data_bytes)
inlineprivate

Definition at line 51 of file ParquetFixedLengthArrayEncoder.h.

References column_desciptor_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), inline_fixed_encoding_null_array_val(), kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, run_benchmark_import::type, and UNREACHABLE.

Referenced by appendNullFixedLengthArray().

51  {
52  auto ti = column_desciptor_.columnType.get_elem_type();
53  if (ti.is_string()) {
54  // TODO: after investigation as to why fixed length arrays with
55  // strings can not represent null arrays, either fix this error
56  // or erase this comment.
57  throw std::runtime_error("Detected a null array being imported into OmniSci '" +
58  column_desciptor_.columnName +
59  "' column which has a fixed length array type of "
60  "dictionary encoded text. Currently "
61  "null arrays for this type of column are not allowed.");
62  }
63  const auto type = ti.get_type();
64  switch (type) {
65  case kBOOLEAN:
66  reinterpret_cast<bool*>(omnisci_data_bytes)[0] =
67  inline_fixed_encoding_null_array_val(ti);
68  break;
69  case kBIGINT:
70  case kNUMERIC:
71  case kDECIMAL:
72  reinterpret_cast<int64_t*>(omnisci_data_bytes)[0] =
73  inline_fixed_encoding_null_array_val(ti);
74  break;
75  case kINT:
76  reinterpret_cast<int32_t*>(omnisci_data_bytes)[0] =
77  inline_fixed_encoding_null_array_val(ti);
78  break;
79  case kSMALLINT:
80  reinterpret_cast<int16_t*>(omnisci_data_bytes)[0] =
81  inline_fixed_encoding_null_array_val(ti);
82  break;
83  case kTINYINT:
84  reinterpret_cast<int8_t*>(omnisci_data_bytes)[0] =
85  inline_fixed_encoding_null_array_val(ti);
86  break;
87  case kFLOAT:
88  reinterpret_cast<float*>(omnisci_data_bytes)[0] = NULL_ARRAY_FLOAT;
89  break;
90  case kDOUBLE:
91  reinterpret_cast<double*>(omnisci_data_bytes)[0] = NULL_ARRAY_DOUBLE;
92  break;
93  case kTIME:
94  case kTIMESTAMP:
95  case kDATE:
96  reinterpret_cast<int64_t*>(omnisci_data_bytes)[0] =
97  inline_fixed_encoding_null_array_val(ti);
98  break;
99  case kTEXT:
100  case kVARCHAR:
101  case kCHAR:
102  case kPOINT:
103  case kLINESTRING:
104  case kPOLYGON:
105  case kMULTIPOLYGON:
106  default:
107  UNREACHABLE();
108  }
109  }
kTIME
Definition: sqltypes.h:51
NULL_ARRAY_DOUBLE
#define NULL_ARRAY_DOUBLE
Definition: sqltypes.h:194
UNREACHABLE
#define UNREACHABLE()
Definition: Logger.h:241
foreign_storage::ParquetFixedLengthArrayEncoder::column_desciptor_
const ColumnDescriptor column_desciptor_
Definition: ParquetFixedLengthArrayEncoder.h:131
kTEXT
Definition: sqltypes.h:54
kDATE
Definition: sqltypes.h:55
kCHAR
Definition: sqltypes.h:43
inline_fixed_encoding_null_array_val
int64_t inline_fixed_encoding_null_array_val(const SQL_TYPE_INFO &ti)
Definition: InlineNullValues.h:180
SQLTypeInfo::get_elem_type
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:624
kINT
Definition: sqltypes.h:47
ColumnDescriptor::columnType
SQLTypeInfo columnType
Definition: ColumnDescriptor.h:35
ColumnDescriptor::columnName
std::string columnName
Definition: ColumnDescriptor.h:33
run_benchmark_import.type
type
Definition: run_benchmark_import.py:89
NULL_ARRAY_FLOAT
#define NULL_ARRAY_FLOAT
Definition: sqltypes.h:193
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

Member Data Documentation

◆ array_element_count_

size_t foreign_storage::ParquetFixedLengthArrayEncoder::array_element_count_
private

Definition at line 132 of file ParquetFixedLengthArrayEncoder.h.

Referenced by appendNullArrayOrCheckArraySize(), and appendNullFixedLengthArray().

◆ column_desciptor_

const ColumnDescriptor foreign_storage::ParquetFixedLengthArrayEncoder::column_desciptor_
private

Definition at line 131 of file ParquetFixedLengthArrayEncoder.h.

Referenced by appendNullArrayOrCheckArraySize(), and setNullFixedLengthArraySentinel().

The documentation for this class was generated from the following file: