#include <ParquetArrayImportEncoder.h>

Inheritance diagram for foreign_storage::ParquetArrayImportEncoder:

Collaboration diagram for foreign_storage::ParquetArrayImportEncoder:

Public Member Functions
	ParquetArrayImportEncoder (Data_Namespace::AbstractBuffer data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor column_desciptor)

void	appendArrayItem (const int64_t encoded_index) override

void	validateAndAppendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices) override

void	resetLastArrayMetadata () override

Public Member Functions inherited from foreign_storage::ParquetArrayEncoder
	ParquetArrayEncoder (Data_Namespace::AbstractBuffer data_buffer, std::shared_ptr< ParquetScalarEncoder > scalar_encoder, const ColumnDescriptor column_desciptor)

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	finalizeRowGroup ()

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

virtual void	disableMetadataStatsValidation () override

virtual void	initializeErrorTracking () override

virtual void	initializeColumnType (const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

RejectedRowIndices	getRejectedRowIndices () const

Protected Member Functions
void	appendArraysToBuffer () override

void	processLastArray () override

Protected Member Functions inherited from foreign_storage::ParquetArrayEncoder
bool	isLastArrayNull () const

bool	isLastArrayEmpty () const

size_t	sizeOfLastArray () const

int8_t *	resizeArrayDataBytes (const size_t additional_num_elements)

bool	isNewArray (const int16_t rep_level) const

int8_t *	encodedDataAtIndex (const size_t index)

void	updateMetadataForAppendedArrayItem (const int64_t encoded_index)

virtual void	encodeAllValues (const int8_t *values, const int64_t values_read)

Private Member Functions
ArrayDatum	convertToArrayDatum (const int8_t *data, const size_t num_elements)

ArrayDatum	getNullArrayDatum ()

void	appendToArrayDatumBuffer ()

void	eraseInvalidIndicesInBuffer (const InvalidRowGroupIndices &invalid_indices) override

Private Attributes
std::vector< bool >	is_valid_item_

TypedParquetStorageBuffer < ArrayDatum > *	array_datum_buffer_

const ColumnDescriptor *	column_descriptor_

size_t	num_array_assembled_

bool	is_invalid_array_

InvalidRowGroupIndices *	invalid_indices_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
size_t	omnisci_data_type_byte_size_

std::shared_ptr < ParquetScalarEncoder >	scalar_encoder_

std::vector< int8_t >	data_buffer_bytes_

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Static Protected Attributes inherited from foreign_storage::ParquetArrayEncoder
static const int16_t	non_null_def_level = 3

static const int16_t	item_null_def_level = 2

static const int16_t	empty_list_def_level = 1

static const int16_t	list_null_def_level = 0

Detailed Description

Definition at line 27 of file ParquetArrayImportEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetArrayImportEncoder::ParquetArrayImportEncoder	(	Data_Namespace::AbstractBuffer *	data_buffer,
		std::shared_ptr< ParquetScalarEncoder >	scalar_encoder,
		const ColumnDescriptor *	column_desciptor
	)

inline

Definition at line 30 of file ParquetArrayImportEncoder.h.

References array_datum_buffer_, and CHECK.

       : ParquetArrayEncoder(data_buffer, scalar_encoder, column_desciptor)
       , array_datum_buffer_(
             dynamic_cast<TypedParquetStorageBuffer<ArrayDatum>*>(data_buffer))
       , column_descriptor_(column_desciptor)
       , num_array_assembled_(0)
       , is_invalid_array_(false)
       , invalid_indices_(nullptr) {
     CHECK(array_datum_buffer_);
   }

Member Function Documentation

void foreign_storage::ParquetArrayImportEncoder::appendArrayItem ( const int64_t encoded_index )

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 43 of file ParquetArrayImportEncoder.h.

References foreign_storage::ParquetArrayEncoder::appendArrayItem(), is_invalid_array_, and is_valid_item_.

                                                              {
     ParquetArrayEncoder::appendArrayItem(encoded_index);
     if (!is_valid_item_[encoded_index]) {
       is_invalid_array_ = true;
     }
   }

Here is the call graph for this function:

void foreign_storage::ParquetArrayImportEncoder::appendArraysToBuffer ( )

inlineoverrideprotectedvirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 76 of file ParquetArrayImportEncoder.h.

                                        {
     // no-op as data is already written to buffer in `processLastArray`
   }

void foreign_storage::ParquetArrayImportEncoder::appendToArrayDatumBuffer ( )

inlineprivate

Definition at line 102 of file ParquetArrayImportEncoder.h.

References foreign_storage::TypedParquetStorageBuffer< Type >::appendElement(), array_datum_buffer_, CHECK, convertToArrayDatum(), foreign_storage::ParquetArrayEncoder::data_buffer_bytes_, getNullArrayDatum(), foreign_storage::ParquetArrayEncoder::isLastArrayEmpty(), foreign_storage::ParquetArrayEncoder::isLastArrayNull(), foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_, and foreign_storage::ParquetArrayEncoder::sizeOfLastArray().

Referenced by processLastArray().

                                   {
     if (isLastArrayNull()) {
       // append a null array offset
       array_datum_buffer_->appendElement(getNullArrayDatum());
     } else if (isLastArrayEmpty()) {
       array_datum_buffer_->appendElement(convertToArrayDatum(nullptr, 0));
     } else {
       CHECK(data_buffer_bytes_.size() ==
             omnisci_data_type_byte_size_ * sizeOfLastArray());
       array_datum_buffer_->appendElement(
           convertToArrayDatum(data_buffer_bytes_.data(), sizeOfLastArray()));
       data_buffer_bytes_
           .clear();  // can clear immediately, only one array buffered at a time
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

ArrayDatum foreign_storage::ParquetArrayImportEncoder::convertToArrayDatum	(	const int8_t *	data,
		const size_t	num_elements
	)

inlineprivate

Definition at line 90 of file ParquetArrayImportEncoder.h.

References foreign_storage::ParquetArrayEncoder::omnisci_data_type_byte_size_.

Referenced by appendToArrayDatumBuffer().

                                                                                 {
     const size_t num_bytes = num_elements * omnisci_data_type_byte_size_;
     std::shared_ptr<int8_t> buffer(new int8_t[num_bytes],
                                    std::default_delete<int8_t[]>());
     memcpy(buffer.get(), data, num_bytes);
     return ArrayDatum(num_bytes, buffer, false);
   }

Here is the caller graph for this function:

void foreign_storage::ParquetArrayImportEncoder::eraseInvalidIndicesInBuffer ( const InvalidRowGroupIndices & invalid_indices )

inlineoverrideprivatevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 118 of file ParquetArrayImportEncoder.h.

References array_datum_buffer_, and foreign_storage::TypedParquetStorageBuffer< Type >::eraseInvalidData().

                                                               {
     if (invalid_indices.empty()) {
       return;
     }
     array_datum_buffer_->eraseInvalidData(invalid_indices);
   }

Here is the call graph for this function:

ArrayDatum foreign_storage::ParquetArrayImportEncoder::getNullArrayDatum ( )

inlineprivate

Definition at line 98 of file ParquetArrayImportEncoder.h.

References column_descriptor_, ColumnDescriptor::columnType, and import_export::ImporterUtils::composeNullArray().

Referenced by appendToArrayDatumBuffer().

                                  {
     return import_export::ImporterUtils::composeNullArray(column_descriptor_->columnType);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetArrayImportEncoder::processLastArray ( )

inlineoverrideprotectedvirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 80 of file ParquetArrayImportEncoder.h.

References appendToArrayDatumBuffer(), CHECK, invalid_indices_, is_invalid_array_, and num_array_assembled_.

                                    {
     appendToArrayDatumBuffer();
     if (is_invalid_array_) {
       CHECK(invalid_indices_);
       invalid_indices_->insert(num_array_assembled_);
     }
     num_array_assembled_++;
   }

Here is the call graph for this function:

void foreign_storage::ParquetArrayImportEncoder::resetLastArrayMetadata ( )

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetArrayEncoder.

Definition at line 70 of file ParquetArrayImportEncoder.h.

References is_invalid_array_, and foreign_storage::ParquetArrayEncoder::resetLastArrayMetadata().

                                          {
     ParquetArrayEncoder::resetLastArrayMetadata();
     is_invalid_array_ = false;
   }

Here is the call graph for this function:

void foreign_storage::ParquetArrayImportEncoder::validateAndAppendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values,
		const SQLTypeInfo &	column_type,
		InvalidRowGroupIndices &	invalid_indices
	)

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 50 of file ParquetArrayImportEncoder.h.

References foreign_storage::ParquetArrayEncoder::appendData(), invalid_indices_, is_valid_item_, and foreign_storage::ParquetArrayEncoder::scalar_encoder_.

                                                                                {
     // validate all elements
     is_valid_item_.assign(values_read, true);
     for (int64_t j = 0; j < values_read; ++j) {
       try {
         scalar_encoder_->validate(values, j, column_type);
       } catch (const std::runtime_error& error) {
         is_valid_item_[j] = false;
       }
     }
     invalid_indices_ = &invalid_indices;  // used in assembly algorithm
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }