OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::ParquetFixedLengthArrayEncoder Class Reference

#include <ParquetFixedLengthArrayEncoder.h>

+ Inheritance diagram for foreign_storage::ParquetFixedLengthArrayEncoder:
+ Collaboration diagram for foreign_storage::ParquetFixedLengthArrayEncoder:

Public Member Functions

 ParquetFixedLengthArrayEncoder (Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
 
std::shared_ptr< ChunkMetadatagetRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override
 
- Public Member Functions inherited from foreign_storage::ParquetArrayEncoder
 ParquetArrayEncoder (Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
 
void appendData (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, const bool is_last_batch, int8_t *values) override
 
- Public Member Functions inherited from foreign_storage::ParquetEncoder
 ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~ParquetEncoder ()=default
 

Protected Member Functions

void processLastArray () override
 
- Protected Member Functions inherited from foreign_storage::ParquetArrayEncoder
virtual void appendArraysToBuffer ()
 
bool isLastArrayNull () const
 
size_t sizeOfLastArray () const
 
int8_t * resizeArrayDataBytes (const size_t additional_num_elements)
 

Private Member Functions

void appendNullFixedLengthArray ()
 
void setNullFixedLengthArraySentinel (int8_t *omnisci_data_bytes)
 
void appendNullArrayOrCheckArraySize ()
 
void throwWrongSizeArray (const size_t size_of_last_array, const size_t array_element_count, const std::string &omnisci_column_name)
 
void throwNullInDictionaryEncodedColumn (const std::string &omnisci_column_name)
 

Private Attributes

const ColumnDescriptor column_desciptor_
 
size_t array_element_count_
 

Additional Inherited Members

- Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr
< ChunkMetadata
createMetadata (const SQLTypeInfo &column_type)
 
static void throwNotNullViolation (const std::string &parquet_column_name)
 
static void validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)
 
- Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
size_t omnisci_data_type_byte_size_
 
std::shared_ptr
< ParquetScalarEncoder
scalar_encoder_
 
std::vector< int8_t > data_buffer_bytes_
 
- Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBufferbuffer_
 
- Static Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
static const int16_t non_null_def_level = 3
 
static const int16_t item_null_def_level = 2
 
static const int16_t list_null_def_level = 0
 

Detailed Description

Definition at line 28 of file ParquetFixedLengthArrayEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetFixedLengthArrayEncoder::ParquetFixedLengthArrayEncoder ( Data_Namespace::AbstractBuffer data_buffer,
std::shared_ptr< ParquetScalarEncoder scalar_encoder,
const ColumnDescriptor column_desciptor 
)
inline

Definition at line 30 of file ParquetFixedLengthArrayEncoder.h.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfo::get_size(), and foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_.

33  : ParquetArrayEncoder(data_buffer, scalar_encoder, column_desciptor)
34  , column_desciptor_(*column_desciptor)
35  , array_element_count_(column_desciptor->columnType.get_size() /
37  CHECK(column_desciptor->columnType.get_size() % omnisci_data_type_byte_size_ == 0);
38  }
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
ParquetArrayEncoder(Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
#define CHECK(condition)
Definition: Logger.h:203
SQLTypeInfo columnType

+ Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize ( )
inlineprivate

Definition at line 119 of file ParquetFixedLengthArrayEncoder.h.

References appendNullFixedLengthArray(), array_element_count_, CHECK, column_desciptor_, ColumnDescriptor::columnName, foreign_storage::ParquetArrayEncoder::isLastArrayNull(), foreign_storage::ParquetArrayEncoder::sizeOfLastArray(), and throwWrongSizeArray().

Referenced by processLastArray().

119  {
120  auto size_of_last_array = sizeOfLastArray();
121  if (!isLastArrayNull()) {
122  if (size_of_last_array != array_element_count_) {
124  size_of_last_array, array_element_count_, column_desciptor_.columnName);
125  }
126  } else {
127  // append a null array sentinel
128  CHECK(size_of_last_array == 0);
130  }
131  }
void throwWrongSizeArray(const size_t size_of_last_array, const size_t array_element_count, const std::string &omnisci_column_name)
#define CHECK(condition)
Definition: Logger.h:203
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray ( )
inlineprivate

Definition at line 55 of file ParquetFixedLengthArrayEncoder.h.

References array_element_count_, i, foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_, foreign_storage::ParquetArrayEncoder::resizeArrayDataBytes(), foreign_storage::ParquetArrayEncoder::scalar_encoder_, and setNullFixedLengthArraySentinel().

Referenced by appendNullArrayOrCheckArraySize().

55  {
56  auto omnisci_data_ptr = resizeArrayDataBytes(array_element_count_);
57  setNullFixedLengthArraySentinel(omnisci_data_ptr);
58  for (size_t i = 1; i < array_element_count_; ++i) {
59  scalar_encoder_->setNull(omnisci_data_ptr + i * omnisci_data_type_byte_size_);
60  }
61  }
int8_t * resizeArrayDataBytes(const size_t additional_num_elements)
std::shared_ptr< ParquetScalarEncoder > scalar_encoder_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr<ChunkMetadata> foreign_storage::ParquetFixedLengthArrayEncoder::getRowGroupMetadata ( const parquet::RowGroupMetaData *  group_metadata,
const int  parquet_column_index,
const SQLTypeInfo column_type 
)
inlineoverridevirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 40 of file ParquetFixedLengthArrayEncoder.h.

References array_element_count_, foreign_storage::ParquetArrayEncoder::getRowGroupMetadata(), and foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_.

43  {
45  group_metadata, parquet_column_index, column_type);
46  metadata->numBytes =
47  omnisci_data_type_byte_size_ * group_metadata->num_rows() * array_element_count_;
48  return metadata;
49  }
std::shared_ptr< ChunkMetadata > getRowGroupMetadata(const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

+ Here is the call graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::processLastArray ( )
inlineoverrideprotectedvirtual

Implements foreign_storage::ParquetArrayEncoder.

Definition at line 52 of file ParquetFixedLengthArrayEncoder.h.

References appendNullArrayOrCheckArraySize().

+ Here is the call graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::setNullFixedLengthArraySentinel ( int8_t *  omnisci_data_bytes)
inlineprivate

Definition at line 63 of file ParquetFixedLengthArrayEncoder.h.

References column_desciptor_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), inline_fixed_encoding_null_array_val(), kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, throwNullInDictionaryEncodedColumn(), run_benchmark_import::type, and UNREACHABLE.

Referenced by appendNullFixedLengthArray().

63  {
65  if (ti.is_string()) {
66  // TODO: after investigation as to why fixed length arrays with
67  // strings can not represent null arrays, either fix this error
68  // or erase this comment.
70  }
71  const auto type = ti.get_type();
72  switch (type) {
73  case kBOOLEAN:
74  reinterpret_cast<bool*>(omnisci_data_bytes)[0] =
76  break;
77  case kBIGINT:
78  case kNUMERIC:
79  case kDECIMAL:
80  reinterpret_cast<int64_t*>(omnisci_data_bytes)[0] =
82  break;
83  case kINT:
84  reinterpret_cast<int32_t*>(omnisci_data_bytes)[0] =
86  break;
87  case kSMALLINT:
88  reinterpret_cast<int16_t*>(omnisci_data_bytes)[0] =
90  break;
91  case kTINYINT:
92  reinterpret_cast<int8_t*>(omnisci_data_bytes)[0] =
94  break;
95  case kFLOAT:
96  reinterpret_cast<float*>(omnisci_data_bytes)[0] = NULL_ARRAY_FLOAT;
97  break;
98  case kDOUBLE:
99  reinterpret_cast<double*>(omnisci_data_bytes)[0] = NULL_ARRAY_DOUBLE;
100  break;
101  case kTIME:
102  case kTIMESTAMP:
103  case kDATE:
104  reinterpret_cast<int64_t*>(omnisci_data_bytes)[0] =
106  break;
107  case kTEXT:
108  case kVARCHAR:
109  case kCHAR:
110  case kPOINT:
111  case kLINESTRING:
112  case kPOLYGON:
113  case kMULTIPOLYGON:
114  default:
115  UNREACHABLE();
116  }
117  }
Definition: sqltypes.h:48
#define UNREACHABLE()
Definition: Logger.h:247
#define NULL_ARRAY_FLOAT
Definition: sqltypes.h:51
Definition: sqltypes.h:52
Definition: sqltypes.h:40
int64_t inline_fixed_encoding_null_array_val(const SQL_TYPE_INFO &ti)
#define NULL_ARRAY_DOUBLE
Definition: sqltypes.h:44
SQLTypeInfo columnType
void throwNullInDictionaryEncodedColumn(const std::string &omnisci_column_name)
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:713
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::throwNullInDictionaryEncodedColumn ( const std::string &  omnisci_column_name)
inlineprivate

Definition at line 146 of file ParquetFixedLengthArrayEncoder.h.

Referenced by setNullFixedLengthArraySentinel().

146  {
147  throw ForeignStorageException("Detected a null array being imported into OmniSci '" +
148  omnisci_column_name +
149  "' column which has a fixed length array type of "
150  "dictionary encoded text. Currently "
151  "null arrays for this type of column are not allowed.");
152  }

+ Here is the caller graph for this function:

void foreign_storage::ParquetFixedLengthArrayEncoder::throwWrongSizeArray ( const size_t  size_of_last_array,
const size_t  array_element_count,
const std::string &  omnisci_column_name 
)
inlineprivate

Definition at line 133 of file ParquetFixedLengthArrayEncoder.h.

References to_string().

Referenced by appendNullArrayOrCheckArraySize().

135  {
136  throw ForeignStorageException("Detected a row with " +
137  std::to_string(size_of_last_array) +
138  " elements being loaded into"
139  " OmniSci column '" +
140  omnisci_column_name +
141  "' which has a fixed length array type,"
142  " expecting " +
143  std::to_string(array_element_count) + " elements.");
144  }
std::string to_string(char const *&&v)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

size_t foreign_storage::ParquetFixedLengthArrayEncoder::array_element_count_
private
const ColumnDescriptor foreign_storage::ParquetFixedLengthArrayEncoder::column_desciptor_
private

The documentation for this class was generated from the following file: