#include <ParquetInPlaceEncoder.h>

Inheritance diagram for foreign_storage::TypedParquetInPlaceEncoder< V, T >:

Collaboration diagram for foreign_storage::TypedParquetInPlaceEncoder< V, T >:

Public Member Functions
	TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer buffer, const ColumnDescriptor column_desciptor, const parquet::ColumnDescriptor *parquet_column_descriptor)

	TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, const bool is_last_batch, int8_t *values) override

void	encodeAndCopyContiguous (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes, const size_t num_elements) override

void	setNull (int8_t *omnisci_data_bytes) override

void	copy (const int8_t omnisci_data_bytes_source, int8_t omnisci_data_bytes_destination) override

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetInPlaceEncoder
	ParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)

Public Member Functions inherited from foreign_storage::ParquetScalarEncoder
	ParquetScalarEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual void	encodeAndCopy (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes)=0

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

Protected Member Functions
virtual bool	encodingIsIdentityForSameTypes () const

std::pair< T, T >	getUnencodedStats (std::shared_ptr< parquet::Statistics > stats) const

Private Member Functions
std::pair< V, V >	getEncodedStats (const parquet::ColumnDescriptor *parquet_column_descriptor, std::shared_ptr< parquet::Statistics > stats)

Static Private Member Functions
static ChunkStats	getUpdatedStats (V &stats_min, V &stats_max, const SQLTypeInfo &column_type)

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes inherited from foreign_storage::ParquetInPlaceEncoder
const size_t	omnisci_data_type_byte_size_

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

Detailed Description

template<typename V, typename T>
class foreign_storage::TypedParquetInPlaceEncoder< V, T >

Definition at line 114 of file ParquetInPlaceEncoder.h.

Constructor & Destructor Documentation

template<typename V, typename T>

foreign_storage::TypedParquetInPlaceEncoder< V, T >::TypedParquetInPlaceEncoder	(	Data_Namespace::AbstractBuffer *	buffer,
		const ColumnDescriptor *	column_desciptor,
		const parquet::ColumnDescriptor *	parquet_column_descriptor
	)

inline

Definition at line 116 of file ParquetInPlaceEncoder.h.

       : ParquetInPlaceEncoder(
             buffer,
             column_desciptor->columnType.get_size(),
             parquet::GetTypeByteSize(parquet_column_descriptor->physical_type())) {}

template<typename V, typename T>

foreign_storage::TypedParquetInPlaceEncoder< V, T >::TypedParquetInPlaceEncoder	(	Data_Namespace::AbstractBuffer *	buffer,
		const size_t	omnisci_data_type_byte_size,
		const size_t	parquet_data_type_byte_size
	)

inline

Definition at line 124 of file ParquetInPlaceEncoder.h.

       : ParquetInPlaceEncoder(buffer,
                               omnisci_data_type_byte_size,
                               parquet_data_type_byte_size) {}

Member Function Documentation

template<typename V, typename T>

void foreign_storage::TypedParquetInPlaceEncoder< V, T >::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		const bool	is_last_batch,
		int8_t *	values
	)

inlineoverridevirtual

This is a specialization of ParquetInPlaceEncoder::appendData for known types that allows for optimization.

See comment for ParquetInPlaceEncoder::appendData for details.

Reimplemented from foreign_storage::ParquetInPlaceEncoder.

Definition at line 137 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::ParquetStringEncoder< V >::appendData().

                                            {
     if (std::is_same<V, T>::value && values_read == levels_read) {
       if (!encodingIsIdentityForSameTypes()) {
         for (int64_t i = 0; i < levels_read; ++i) {
           encodeAndCopy(values + i * omnisci_data_type_byte_size_,
                         values + i * omnisci_data_type_byte_size_);
         }
       }
       buffer_->append(values, levels_read * omnisci_data_type_byte_size_);
     } else {
       ParquetInPlaceEncoder::appendData(
           def_levels, rep_levels, values_read, levels_read, is_last_batch, values);
     }
   }

Here is the caller graph for this function:

template<typename V, typename T>

void foreign_storage::TypedParquetInPlaceEncoder< V, T >::copy	(	const int8_t *	omnisci_data_bytes_source,
		int8_t *	omnisci_data_bytes_destination
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 173 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::ParquetStringEncoder< V >::encodeAndCopy().

                                                              {
     const auto& omnisci_data_value_source =
         reinterpret_cast<const V*>(omnisci_data_bytes_source)[0];
     auto& omnisci_data_value_destination =
         reinterpret_cast<V*>(omnisci_data_bytes_destination)[0];
     omnisci_data_value_destination = omnisci_data_value_source;
   }

Here is the caller graph for this function:

template<typename V, typename T>

void foreign_storage::TypedParquetInPlaceEncoder< V, T >::encodeAndCopyContiguous	(	const int8_t *	parquet_data_bytes,
		int8_t *	omnisci_data_bytes,
		const size_t	num_elements
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 157 of file ParquetInPlaceEncoder.h.

                                                                    {
     auto parquet_data_ptr = reinterpret_cast<const T*>(parquet_data_bytes);
     auto omnisci_data_ptr = reinterpret_cast<V*>(omnisci_data_bytes);
     for (size_t i = 0; i < num_elements; ++i) {
       encodeAndCopy(reinterpret_cast<const int8_t*>(&parquet_data_ptr[i]),
                     reinterpret_cast<int8_t*>(&omnisci_data_ptr[i]));
     }
   }

template<typename V, typename T>

virtual bool foreign_storage::TypedParquetInPlaceEncoder< V, T >::encodingIsIdentityForSameTypes ( ) const

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetStringEncoder< V >, foreign_storage::ParquetFixedLengthEncoder< V, T >, and foreign_storage::ParquetDecimalEncoder< V, T >.

Definition at line 228 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t >::appendData().

228 { return false; }

Here is the caller graph for this function:

template<typename V, typename T>

std::pair<V, V> foreign_storage::TypedParquetInPlaceEncoder< V, T >::getEncodedStats	(	const parquet::ColumnDescriptor *	parquet_column_descriptor,
		std::shared_ptr< parquet::Statistics >	stats
	)

inlineprivate

Definition at line 260 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t >::getRowGroupMetadata().

                                               {
     V stats_min, stats_max;
     auto min_string = stats->EncodeMin();
     auto max_string = stats->EncodeMax();
     if (parquet_column_descriptor->physical_type() ==
         parquet::Type::FIXED_LEN_BYTE_ARRAY) {
       parquet::FixedLenByteArray min_byte_array, max_byte_array;
       min_byte_array.ptr = reinterpret_cast<const uint8_t*>(min_string.data());
       max_byte_array.ptr = reinterpret_cast<const uint8_t*>(max_string.data());
       encodeAndCopy(reinterpret_cast<int8_t*>(&min_byte_array),
                     reinterpret_cast<int8_t*>(&stats_min));
       encodeAndCopy(reinterpret_cast<int8_t*>(&max_byte_array),
                     reinterpret_cast<int8_t*>(&stats_max));
     } else if (parquet_column_descriptor->physical_type() == parquet::Type::BYTE_ARRAY) {
       parquet::ByteArray min_byte_array, max_byte_array;
       min_byte_array.ptr = reinterpret_cast<const uint8_t*>(min_string.data());
       min_byte_array.len = min_string.length();
       max_byte_array.ptr = reinterpret_cast<const uint8_t*>(max_string.data());
       max_byte_array.len = max_string.length();
       encodeAndCopy(reinterpret_cast<int8_t*>(&min_byte_array),
                     reinterpret_cast<int8_t*>(&stats_min));
       encodeAndCopy(reinterpret_cast<int8_t*>(&max_byte_array),
                     reinterpret_cast<int8_t*>(&stats_max));
     } else {
       encodeAndCopy(reinterpret_cast<int8_t*>(min_string.data()),
                     reinterpret_cast<int8_t*>(&stats_min));
       encodeAndCopy(reinterpret_cast<int8_t*>(max_string.data()),
                     reinterpret_cast<int8_t*>(&stats_max));
     }
     return {stats_min, stats_max};
   }

Here is the caller graph for this function:

template<typename V, typename T>

std::shared_ptr<ChunkMetadata> foreign_storage::TypedParquetInPlaceEncoder< V, T >::getRowGroupMetadata	(	const parquet::RowGroupMetaData *	group_metadata,
		const int	parquet_column_index,
		const SQLTypeInfo &	column_type
	)

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Definition at line 182 of file ParquetInPlaceEncoder.h.

                                                {
     auto metadata = ParquetEncoder::createMetadata(column_type);
     auto column_metadata = group_metadata->ColumnChunk(parquet_column_index);
 
     // update statistics
     auto parquet_column_descriptor =
         group_metadata->schema()->Column(parquet_column_index);
     auto stats = validate_and_get_column_metadata_statistics(column_metadata.get());
     if (stats->HasMinMax()) {
       // validate statistics if validation applicable as part of encoding
       if (auto parquet_scalar_validator = dynamic_cast<ParquetMetadataValidator*>(this)) {
         try {
           parquet_scalar_validator->validate(
               stats, column_type.is_array() ? column_type.get_elem_type() : column_type);
         } catch (const std::exception& e) {
           std::stringstream error_message;
           error_message << e.what() << " Error validating statistics of Parquet column '"
                         << group_metadata->schema()->Column(parquet_column_index)->name()
                         << "'";
           throw std::runtime_error(error_message.str());
         }
       }
 
       auto [stats_min, stats_max] = getEncodedStats(parquet_column_descriptor, stats);
       auto updated_chunk_stats = getUpdatedStats(stats_min, stats_max, column_type);
       metadata->fillChunkStats(updated_chunk_stats.min,
                                updated_chunk_stats.max,
                                metadata->chunkStats.has_nulls);
     }
     auto null_count = stats->null_count();
     validateNullCount(group_metadata->schema()->Column(parquet_column_index)->name(),
                       null_count,
                       column_type);
     metadata->chunkStats.has_nulls = null_count > 0;
 
     // update sizing
     metadata->numBytes = omnisci_data_type_byte_size_ * column_metadata->num_values();
     metadata->numElements = group_metadata->num_rows();
 
     return metadata;
   }

template<typename V, typename T>

std::pair<T, T> foreign_storage::TypedParquetInPlaceEncoder< V, T >::getUnencodedStats ( std::shared_ptr< parquet::Statistics > stats ) const

inlineprotected

Definition at line 230 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::ParquetDateInSecondsEncoder::validate(), foreign_storage::ParquetTimestampEncoder< V, T, conversion_denominator >::validate(), foreign_storage::ParquetUnsignedFixedLengthEncoder< V, T, U >::validate(), and foreign_storage::ParquetFixedLengthEncoder< V, T >::validateIntegralOrFloatingPointValue().

                                                                                 {
     T stats_min = reinterpret_cast<T*>(stats->EncodeMin().data())[0];
     T stats_max = reinterpret_cast<T*>(stats->EncodeMax().data())[0];
     return {stats_min, stats_max};
   }

Here is the caller graph for this function:

template<typename V, typename T>

static ChunkStats foreign_storage::TypedParquetInPlaceEncoder< V, T >::getUpdatedStats	(	V &	stats_min,
		V &	stats_max,
		const SQLTypeInfo &	column_type
	)

inlinestaticprivate

Definition at line 237 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t >::getRowGroupMetadata().

                                                                     {
     ForeignStorageBuffer buffer;
     buffer.initEncoder(column_type);
     auto encoder = buffer.getEncoder();
 
     if (column_type.is_array()) {
       ArrayDatum min_datum(
           sizeof(V), reinterpret_cast<int8_t*>(&stats_min), false, DoNothingDeleter());
       ArrayDatum max_datum(
           sizeof(V), reinterpret_cast<int8_t*>(&stats_max), false, DoNothingDeleter());
       std::vector<ArrayDatum> min_max_datums{min_datum, max_datum};
       encoder->updateStats(&min_max_datums, 0, 1);
     } else {
       encoder->updateStats(reinterpret_cast<int8_t*>(&stats_min), 1);
       encoder->updateStats(reinterpret_cast<int8_t*>(&stats_max), 1);
     }
     auto updated_chunk_stats_metadata = std::make_shared<ChunkMetadata>();
     encoder->getMetadata(updated_chunk_stats_metadata);
     return updated_chunk_stats_metadata->chunkStats;
   }

Here is the caller graph for this function:

template<typename V, typename T>

void foreign_storage::TypedParquetInPlaceEncoder< V, T >::setNull ( int8_t * omnisci_data_bytes )

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 168 of file ParquetInPlaceEncoder.h.

                                                     {
     auto& omnisci_data_value = reinterpret_cast<V*>(omnisci_data_bytes)[0];
     omnisci_data_value = get_null_value<V>();
   }

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetInPlaceEncoder.h

Public Member Functions

Protected Member Functions

Private Member Functions

Static Private Member Functions

Additional Inherited Members

Detailed Description

template<typename V, typename T> class foreign_storage::TypedParquetInPlaceEncoder< V, T >

Constructor & Destructor Documentation

Member Function Documentation

template<typename V, typename T>
class foreign_storage::TypedParquetInPlaceEncoder< V, T >