foreign_storage::ParquetVariableLengthArrayEncoder Class Reference

#include <ParquetVariableLengthArrayEncoder.h>

Inheritance diagram for foreign_storage::ParquetVariableLengthArrayEncoder:

Collaboration diagram for foreign_storage::ParquetVariableLengthArrayEncoder:

Public Member Functions
	ParquetVariableLengthArrayEncoder (Data_Namespace::AbstractBuffer *data_buffer, Data_Namespace::AbstractBuffer *index_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
void	appendData (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, const bool is_last_batch, int8_t *values) override
Public Member Functions inherited from foreign_storage::ParquetArrayEncoder
	ParquetArrayEncoder (Data_Namespace::AbstractBuffer *data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor *column_desciptor)
std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override
Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)
virtual	~ParquetEncoder ()=default

Protected Member Functions
void	appendArraysToBuffer () override
void	processLastArray () override
Protected Member Functions inherited from foreign_storage::ParquetArrayEncoder
bool	isLastArrayNull () const
size_t	sizeOfLastArray () const
int8_t *	resizeArrayDataBytes (const size_t additional_num_elements)

Private Member Functions
void	setFirstOffsetForBuffer (const int16_t def_level)
void	appendLastArrayOffset ()

Private Attributes
Data_Namespace::AbstractBuffer *	index_buffer_
std::vector< ArrayOffsetT >	offsets_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)
static void	throwNotNullViolation (const std::string &parquet_column_name)
static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)
Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
size_t	omnisci_data_type_byte_size_
std::shared_ptr < ParquetScalarEncoder >	scalar_encoder_
std::vector< int8_t >	data_buffer_bytes_
Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_
Static Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
static const int16_t	non_null_def_level = 3
static const int16_t	item_null_def_level = 2
static const int16_t	list_null_def_level = 0

Detailed Description

Definition at line 25 of file ParquetVariableLengthArrayEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetVariableLengthArrayEncoder::ParquetVariableLengthArrayEncoder ( Data_Namespace::AbstractBuffer *  data_buffer, Data_Namespace::AbstractBuffer *  index_buffer, std::shared_ptr< ParquetScalarEncoder >  scalar_encoder, const ColumnDescriptor *  column_desciptor  )

inline

Definition at line 27 of file ParquetVariableLengthArrayEncoder.h.

      : ParquetArrayEncoder(data_buffer, scalar_encoder, column_desciptor)
      , index_buffer_(index_buffer) {}

Member Function Documentation

void foreign_storage::ParquetVariableLengthArrayEncoder::appendArraysToBuffer ( )

inlineoverrideprotectedvirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 47 of file ParquetVariableLengthArrayEncoder.h.

References Data_Namespace::AbstractBuffer::append(), foreign_storage::ParquetArrayEncoder::appendArraysToBuffer(), index_buffer_, and offsets_.

                                       {
    index_buffer_->append(reinterpret_cast<int8_t*>(offsets_.data()),
                          offsets_.size() * sizeof(ArrayOffsetT));
    offsets_.clear();
    ParquetArrayEncoder::appendArraysToBuffer();
  }

Here is the call graph for this function:

void foreign_storage::ParquetVariableLengthArrayEncoder::appendData ( const int16_t *  def_levels, const int16_t *  rep_levels, const int64_t  values_read, const int64_t  levels_read, const bool  is_last_batch, int8_t *  values  )

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 34 of file ParquetVariableLengthArrayEncoder.h.

References foreign_storage::ParquetArrayEncoder::appendData(), CHECK, and setFirstOffsetForBuffer().

                                           {
    CHECK(levels_read > 0);
    setFirstOffsetForBuffer(def_levels[0]);
    ParquetArrayEncoder::appendData(
        def_levels, rep_levels, values_read, levels_read, is_last_batch, values);
  }

Here is the call graph for this function:

void foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset ( )

inlineprivate

Definition at line 75 of file ParquetVariableLengthArrayEncoder.h.

References foreign_storage::ParquetEncoder::buffer_, foreign_storage::ParquetArrayEncoder::data_buffer_bytes_, foreign_storage::ParquetArrayEncoder::isLastArrayNull(), offsets_, and Data_Namespace::AbstractBuffer::size().

Referenced by processLastArray().

                               {
    int64_t last_offset = buffer_->size() + data_buffer_bytes_.size();
    if (!isLastArrayNull()) {
      // append array data offset
      offsets_.push_back(last_offset);
    } else {
      // append a null array offset
      offsets_.push_back(-last_offset);
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetVariableLengthArrayEncoder::processLastArray ( )

inlineoverrideprotectedvirtual

Implements foreign_storage::ParquetArrayEncoder.

Definition at line 54 of file ParquetVariableLengthArrayEncoder.h.

References appendLastArrayOffset().

{ appendLastArrayOffset(); }

Here is the call graph for this function:

void foreign_storage::ParquetVariableLengthArrayEncoder::setFirstOffsetForBuffer ( const int16_t  def_level )

inlineprivate

Definition at line 57 of file ParquetVariableLengthArrayEncoder.h.

References foreign_storage::ParquetEncoder::buffer_, foreign_storage::ParquetArrayEncoder::data_buffer_bytes_, ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE, foreign_storage::ParquetArrayEncoder::list_null_def_level, offsets_, and Data_Namespace::AbstractBuffer::size().

Referenced by appendData().

                                                        {
    if (data_buffer_bytes_.size() == 0 && buffer_->size() == 0) {  // first  element
      if (def_level == ParquetArrayEncoder::list_null_def_level) {
        // OmniSci variable array types have a special encoding for chunks in
        // which the first array is null: the first `DEFAULT_NULL_PADDING_SIZE`
        // bytes of the chunk are filled and the offset is set appropriately.
        // Ostensibly, this is done to allow marking a null array by negating
        // a non-zero value.
        offsets_.push_back(ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE);
        std::vector<int8_t> zero_bytes(ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE, 0);
        data_buffer_bytes_.insert(
            data_buffer_bytes_.end(), zero_bytes.begin(), zero_bytes.end());
      } else {
        offsets_.push_back(0);
      }
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

Data_Namespace::AbstractBuffer* foreign_storage::ParquetVariableLengthArrayEncoder::index_buffer_

private

Definition at line 86 of file ParquetVariableLengthArrayEncoder.h.

Referenced by appendArraysToBuffer().

std::vector<ArrayOffsetT> foreign_storage::ParquetVariableLengthArrayEncoder::offsets_

private

Definition at line 87 of file ParquetVariableLengthArrayEncoder.h.

Referenced by appendArraysToBuffer(), appendLastArrayOffset(), and setFirstOffsetForBuffer().

---

The documentation for this class was generated from the following file:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetVariableLengthArrayEncoder.h