#include <ParquetArrayEncoder.h>

Inheritance diagram for foreign_storage::ParquetArrayEncoder:

Collaboration diagram for foreign_storage::ParquetArrayEncoder:

Public Member Functions
	ParquetArrayEncoder (Data_Namespace::AbstractBuffer data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor column_desciptor)

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, const bool is_last_batch, int8_t *values) override

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

Protected Member Functions
virtual void	processLastArray ()=0

virtual void	appendArraysToBuffer ()

bool	isLastArrayNull () const

size_t	sizeOfLastArray () const

int8_t *	resizeArrayDataBytes (const size_t additional_num_elements)

Protected Attributes
size_t	omnisci_data_type_byte_size_

std::shared_ptr < ParquetScalarEncoder >	scalar_encoder_

std::vector< int8_t >	data_buffer_bytes_

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

Static Protected Attributes
static const int16_t	non_null_def_level = 3

static const int16_t	item_null_def_level = 2

static const int16_t	list_null_def_level = 0

Private Member Functions
void	finalizeRowGroup ()

void	resetLastArrayMetadata ()

bool	isNewArray (const int16_t rep_level) const

void	processArrayItem (const int16_t def_level, int64_t &encoded_index)

void	encodeAllValues (const int8_t *values, const int64_t values_read)

void	markArrayAsNull ()

void	appendArrayItem (const int64_t encoded_index)

void	appendNullArrayItem ()

Private Attributes
std::vector< int8_t >	encode_buffer_

bool	has_assembly_started_

bool	is_null_array_

size_t	num_elements_in_array_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Detailed Description

Definition at line 23 of file ParquetArrayEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetArrayEncoder::ParquetArrayEncoder	(	Data_Namespace::AbstractBuffer *	data_buffer,
		std::shared_ptr< ParquetScalarEncoder >	scalar_encoder,
		const ColumnDescriptor *	column_desciptor
	)

inline

Definition at line 25 of file ParquetArrayEncoder.h.

       : ParquetEncoder(data_buffer)
       , omnisci_data_type_byte_size_(
             column_desciptor->columnType.get_elem_type().get_size())
       , scalar_encoder_(scalar_encoder)
       , has_assembly_started_(false)
       , is_null_array_(false)
       , num_elements_in_array_(0) {}

Member Function Documentation

void foreign_storage::ParquetArrayEncoder::appendArrayItem ( const int64_t encoded_index )

inlineprivate

Definition at line 137 of file ParquetArrayEncoder.h.

References encode_buffer_, num_elements_in_array_, omnisci_data_type_byte_size_, resizeArrayDataBytes(), and scalar_encoder_.

Referenced by processArrayItem().

                                                     {
     auto omnisci_data_ptr = resizeArrayDataBytes(1);
     scalar_encoder_->copy(
         encode_buffer_.data() + (encoded_index)*omnisci_data_type_byte_size_,
         omnisci_data_ptr);
     num_elements_in_array_++;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::appendArraysToBuffer ( )

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetVariableLengthArrayEncoder.

Definition at line 73 of file ParquetArrayEncoder.h.

References Data_Namespace::AbstractBuffer::append(), foreign_storage::ParquetEncoder::buffer_, and data_buffer_bytes_.

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendArraysToBuffer(), and finalizeRowGroup().

                                       {
     buffer_->append(data_buffer_bytes_.data(), data_buffer_bytes_.size());
     data_buffer_bytes_.clear();
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		const bool	is_last_batch,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Reimplemented in foreign_storage::ParquetVariableLengthArrayEncoder.

Definition at line 36 of file ParquetArrayEncoder.h.

References CHECK, encodeAllValues(), finalizeRowGroup(), isNewArray(), processArrayItem(), processLastArray(), and resetLastArrayMetadata().

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendData().

                                            {
     CHECK(levels_read > 0);
 
     // encode all values in the temporary in-memory `encode_buffer_`, doing
     // this encoding as a batch rather than element-wise exposes opportunities
     // for performance optimization for certain scalar types
     encodeAllValues(values, values_read);
 
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       if (isNewArray(rep_levels[i])) {
         processLastArray();
         resetLastArrayMetadata();
       }
       processArrayItem(def_levels[i], j);
     }
 
     if (is_last_batch) {
       finalizeRowGroup();
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::appendNullArrayItem ( )

inlineprivate

Definition at line 145 of file ParquetArrayEncoder.h.

References num_elements_in_array_, resizeArrayDataBytes(), and scalar_encoder_.

Referenced by processArrayItem().

                              {
     scalar_encoder_->setNull(resizeArrayDataBytes(1));
     num_elements_in_array_++;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::encodeAllValues	(	const int8_t *	values,
		const int64_t	values_read
	)

inlineprivate

Definition at line 130 of file ParquetArrayEncoder.h.

References encode_buffer_, omnisci_data_type_byte_size_, and scalar_encoder_.

Referenced by appendData().

                                                                         {
     encode_buffer_.resize(values_read * omnisci_data_type_byte_size_);
     scalar_encoder_->encodeAndCopyContiguous(values, encode_buffer_.data(), values_read);
   }

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::finalizeRowGroup ( )

inlineprivate

Definition at line 99 of file ParquetArrayEncoder.h.

References appendArraysToBuffer(), has_assembly_started_, processLastArray(), and resetLastArrayMetadata().

Referenced by appendData().

                           {
     processLastArray();
     resetLastArrayMetadata();
     appendArraysToBuffer();
     has_assembly_started_ = false;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ChunkMetadata> foreign_storage::ParquetArrayEncoder::getRowGroupMetadata	(	const parquet::RowGroupMetaData *	group_metadata,
		const int	parquet_column_index,
		const SQLTypeInfo &	column_type
	)

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Reimplemented in foreign_storage::ParquetFixedLengthArrayEncoder.

Definition at line 62 of file ParquetArrayEncoder.h.

References scalar_encoder_.

Referenced by foreign_storage::ParquetFixedLengthArrayEncoder::getRowGroupMetadata().

                                                {
     return scalar_encoder_->getRowGroupMetadata(
         group_metadata, parquet_column_index, column_type);
   }

Here is the caller graph for this function:

bool foreign_storage::ParquetArrayEncoder::isLastArrayNull ( ) const

inlineprotected

Definition at line 78 of file ParquetArrayEncoder.h.

References is_null_array_.

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset(), and foreign_storage::ParquetFixedLengthArrayEncoder::appendNullArrayOrCheckArraySize().

78 { return is_null_array_; }

foreign_storage::ParquetArrayEncoder::is_null_array_

bool is_null_array_

Definition: ParquetArrayEncoder.h:152

Here is the caller graph for this function:

bool foreign_storage::ParquetArrayEncoder::isNewArray ( const int16_t rep_level ) const

inlineprivate

Definition at line 111 of file ParquetArrayEncoder.h.

References has_assembly_started_.

Referenced by appendData().

                                                  {
     return rep_level == 0 && has_assembly_started_;
   }

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::markArrayAsNull ( )

inlineprivate

Definition at line 135 of file ParquetArrayEncoder.h.

References is_null_array_.

Referenced by processArrayItem().

135 { is_null_array_ = true; }

foreign_storage::ParquetArrayEncoder::is_null_array_

bool is_null_array_

Definition: ParquetArrayEncoder.h:152

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::processArrayItem	(	const int16_t	def_level,
		int64_t &	encoded_index
	)

inlineprivate

Definition at line 115 of file ParquetArrayEncoder.h.

References appendArrayItem(), appendNullArrayItem(), has_assembly_started_, item_null_def_level, list_null_def_level, markArrayAsNull(), non_null_def_level, and UNREACHABLE.

Referenced by appendData().

                                                                          {
     has_assembly_started_ = true;
     if (def_level == non_null_def_level) {
       // push back a scalar element to in-memory data buffer
       appendArrayItem(encoded_index++);
     } else if (def_level == item_null_def_level) {
       // push back a scalar null to in-memory data buffer
       appendNullArrayItem();
     } else if (def_level == list_null_def_level) {
       markArrayAsNull();
     } else {
       UNREACHABLE();
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual void foreign_storage::ParquetArrayEncoder::processLastArray ( )

protectedpure virtual

Implemented in foreign_storage::ParquetFixedLengthArrayEncoder, and foreign_storage::ParquetVariableLengthArrayEncoder.

Referenced by appendData(), and finalizeRowGroup().

Here is the caller graph for this function:

void foreign_storage::ParquetArrayEncoder::resetLastArrayMetadata ( )

inlineprivate

Definition at line 106 of file ParquetArrayEncoder.h.

References is_null_array_, and num_elements_in_array_.

Referenced by appendData(), and finalizeRowGroup().

                                 {
     is_null_array_ = false;
     num_elements_in_array_ = 0;
   }

Here is the caller graph for this function:

int8_t* foreign_storage::ParquetArrayEncoder::resizeArrayDataBytes ( const size_t additional_num_elements )

inlineprotected

Definition at line 82 of file ParquetArrayEncoder.h.

References data_buffer_bytes_, and omnisci_data_type_byte_size_.

Referenced by appendArrayItem(), appendNullArrayItem(), and foreign_storage::ParquetFixedLengthArrayEncoder::appendNullFixedLengthArray().

                                                                      {
     auto current_data_byte_size = data_buffer_bytes_.size();
     data_buffer_bytes_.resize(current_data_byte_size +
                               additional_num_elements * omnisci_data_type_byte_size_);
     return data_buffer_bytes_.data() + current_data_byte_size;
   }