OmniSciDB
bf83d84833
|
#include <ParquetArrayEncoder.h>
Public Member Functions | |
ParquetArrayEncoder (Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor) | |
void | appendData (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, const bool is_last_batch, int8_t *values) override |
std::shared_ptr< ChunkMetadata > | getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override |
![]() | |
ParquetEncoder (Data_Namespace::AbstractBuffer *buffer) | |
virtual | ~ParquetEncoder ()=default |
Protected Member Functions | |
virtual void | processLastArray ()=0 |
virtual void | appendArraysToBuffer () |
bool | isLastArrayNull () const |
size_t | sizeOfLastArray () const |
int8_t * | resizeArrayDataBytes (const size_t additional_num_elements) |
Protected Attributes | |
size_t | omnisci_data_type_byte_size_ |
std::shared_ptr < ParquetScalarEncoder > | scalar_encoder_ |
std::vector< int8_t > | data_buffer_bytes_ |
![]() | |
Data_Namespace::AbstractBuffer * | buffer_ |
Static Protected Attributes | |
static const int16_t | non_null_def_level = 3 |
static const int16_t | item_null_def_level = 2 |
static const int16_t | list_null_def_level = 0 |
Private Member Functions | |
void | finalizeRowGroup () |
void | resetLastArrayMetadata () |
bool | isNewArray (const int16_t rep_level) const |
void | processArrayItem (const int16_t def_level, int64_t &encoded_index) |
void | encodeAllValues (const int8_t *values, const int64_t values_read) |
void | markArrayAsNull () |
void | appendArrayItem (const int64_t encoded_index) |
void | appendNullArrayItem () |
Private Attributes | |
std::vector< int8_t > | encode_buffer_ |
bool | has_assembly_started_ |
bool | is_null_array_ |
size_t | num_elements_in_array_ |
Additional Inherited Members | |
![]() | |
static std::shared_ptr < ChunkMetadata > | createMetadata (const SQLTypeInfo &column_type) |
static void | throwNotNullViolation (const std::string &parquet_column_name) |
static void | validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type) |
Definition at line 23 of file ParquetArrayEncoder.h.
|
inline |
Definition at line 25 of file ParquetArrayEncoder.h.
|
inlineprivate |
Definition at line 137 of file ParquetArrayEncoder.h.
References encode_buffer_, num_elements_in_array_, omnisci_data_type_byte_size_, resizeArrayDataBytes(), and scalar_encoder_.
Referenced by processArrayItem().
|
inlineprotectedvirtual |
Reimplemented in foreign_storage::ParquetVariableLengthArrayEncoder.
Definition at line 73 of file ParquetArrayEncoder.h.
References Data_Namespace::AbstractBuffer::append(), foreign_storage::ParquetEncoder::buffer_, and data_buffer_bytes_.
Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendArraysToBuffer(), and finalizeRowGroup().
|
inlineoverridevirtual |
Implements foreign_storage::ParquetEncoder.
Reimplemented in foreign_storage::ParquetVariableLengthArrayEncoder.
Definition at line 36 of file ParquetArrayEncoder.h.
References CHECK, encodeAllValues(), finalizeRowGroup(), isNewArray(), processArrayItem(), processLastArray(), and resetLastArrayMetadata().
Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendData().
|
inlineprivate |
Definition at line 145 of file ParquetArrayEncoder.h.
References num_elements_in_array_, resizeArrayDataBytes(), and scalar_encoder_.
Referenced by processArrayItem().
|
inlineprivate |
Definition at line 130 of file ParquetArrayEncoder.h.
References encode_buffer_, omnisci_data_type_byte_size_, and scalar_encoder_.
Referenced by appendData().
|
inlineprivate |
Definition at line 99 of file ParquetArrayEncoder.h.
References appendArraysToBuffer(), has_assembly_started_, processLastArray(), and resetLastArrayMetadata().
Referenced by appendData().
|
inlineoverridevirtual |
Reimplemented from foreign_storage::ParquetEncoder.
Reimplemented in foreign_storage::ParquetFixedLengthArrayEncoder.
Definition at line 62 of file ParquetArrayEncoder.h.
References scalar_encoder_.
Referenced by foreign_storage::ParquetFixedLengthArrayEncoder::getRowGroupMetadata().
|
inlineprotected |
Definition at line 78 of file ParquetArrayEncoder.h.
References is_null_array_.
Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset(), and foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize().
|
inlineprivate |
Definition at line 111 of file ParquetArrayEncoder.h.
References has_assembly_started_.
Referenced by appendData().
|
inlineprivate |
Definition at line 135 of file ParquetArrayEncoder.h.
References is_null_array_.
Referenced by processArrayItem().
|
inlineprivate |
Definition at line 115 of file ParquetArrayEncoder.h.
References appendArrayItem(), appendNullArrayItem(), has_assembly_started_, item_null_def_level, list_null_def_level, markArrayAsNull(), non_null_def_level, and UNREACHABLE.
Referenced by appendData().
|
protectedpure virtual |
Implemented in foreign_storage::ParquetFixedLengthArrayEncoder, and foreign_storage::ParquetVariableLengthArrayEncoder.
Referenced by appendData(), and finalizeRowGroup().
|
inlineprivate |
Definition at line 106 of file ParquetArrayEncoder.h.
References is_null_array_, and num_elements_in_array_.
Referenced by appendData(), and finalizeRowGroup().
|
inlineprotected |
Definition at line 82 of file ParquetArrayEncoder.h.
References data_buffer_bytes_, and omnisci_data_type_byte_size_.
Referenced by appendArrayItem(), appendNullArrayItem(), and foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray().
|
inlineprotected |
Definition at line 80 of file ParquetArrayEncoder.h.
References num_elements_in_array_.
Referenced by foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize().
|
protected |
Definition at line 91 of file ParquetArrayEncoder.h.
Referenced by appendArraysToBuffer(), foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset(), resizeArrayDataBytes(), and foreign_storage::ParquetVariableLengthArrayEncoder::setFirstOffsetForBuffer().
|
private |
Definition at line 150 of file ParquetArrayEncoder.h.
Referenced by appendArrayItem(), and encodeAllValues().
|
private |
Definition at line 151 of file ParquetArrayEncoder.h.
Referenced by finalizeRowGroup(), isNewArray(), and processArrayItem().
|
private |
Definition at line 152 of file ParquetArrayEncoder.h.
Referenced by isLastArrayNull(), markArrayAsNull(), and resetLastArrayMetadata().
|
staticprotected |
Definition at line 95 of file ParquetArrayEncoder.h.
Referenced by processArrayItem().
|
staticprotected |
Definition at line 96 of file ParquetArrayEncoder.h.
Referenced by processArrayItem(), and foreign_storage::ParquetVariableLengthArrayEncoder::setFirstOffsetForBuffer().
|
staticprotected |
Definition at line 94 of file ParquetArrayEncoder.h.
Referenced by processArrayItem().
|
private |
Definition at line 153 of file ParquetArrayEncoder.h.
Referenced by appendArrayItem(), appendNullArrayItem(), resetLastArrayMetadata(), and sizeOfLastArray().
|
protected |
Definition at line 89 of file ParquetArrayEncoder.h.
Referenced by appendArrayItem(), foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray(), encodeAllValues(), foreign_storage::ParquetFixedLengthArrayEncoder::getRowGroupMetadata(), foreign_storage::ParquetFixedLengthArrayEncoder::ParquetFixedLengthArrayEncoder(), and resizeArrayDataBytes().
|
protected |
Definition at line 90 of file ParquetArrayEncoder.h.
Referenced by appendArrayItem(), appendNullArrayItem(), foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray(), encodeAllValues(), and getRowGroupMetadata().