#include <ParquetInPlaceEncoder.h>

Inheritance diagram for foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >:

Collaboration diagram for foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >:

Public Member Functions
	TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer buffer, const ColumnDescriptor column_desciptor, const parquet::ColumnDescriptor *parquet_column_descriptor)

	TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)

void	validate (const int8_t *parquet_data, const int64_t j, const SQLTypeInfo &column_type) const override

std::string	integralTypeToString (const V &element) const

bool	isIntegralType (const SQLTypeInfo &type) const

std::string	elementToString (const V &element) const

std::string	encodedDataToString (const int8_t *bytes) const override

void	setDetectBufferConverterType ()

void	validateUsingEncodersColumnType (const int8_t *parquet_data, const int64_t j) const override

void	reserve (const size_t num_append_elements) override

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	validateAndAppendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices) override

void	eraseInvalidIndicesInBuffer (const InvalidRowGroupIndices &invalid_indices) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	encodeAndCopyContiguous (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes, const size_t num_elements) override

void	setNull (int8_t *omnisci_data_bytes) override

void	copy (const int8_t omnisci_data_bytes_source, int8_t omnisci_data_bytes_destination) override

std::shared_ptr< ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override

Public Member Functions inherited from foreign_storage::ParquetInPlaceEncoder
	ParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)

Public Member Functions inherited from foreign_storage::ParquetScalarEncoder
	ParquetScalarEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual void	encodeAndCopy (const int8_t parquet_data_bytes, int8_t omnisci_data_bytes)=0

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

RejectedRowIndices	getRejectedRowIndices () const

virtual void	disableMetadataStatsValidation ()

virtual void	initializeErrorTracking (const SQLTypeInfo &column_type)

Protected Member Functions
virtual bool	encodingIsIdentityForSameTypes () const

std::pair< T, T >	getUnencodedStats (std::shared_ptr< parquet::Statistics > stats) const

Private Member Functions
std::pair< V, V >	getEncodedStats (const parquet::ColumnDescriptor *parquet_column_descriptor, std::shared_ptr< parquet::Statistics > stats)

Static Private Member Functions
static ChunkStats	getUpdatedStats (V &stats_min, V &stats_max, const SQLTypeInfo &column_type)

Private Attributes
int64_t	current_batch_offset_ = 0

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes inherited from foreign_storage::ParquetInPlaceEncoder
const size_t	omnisci_data_type_byte_size_

const size_t	parquet_data_type_byte_size_

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Detailed Description

template<typename V, typename T, typename NullType = V>
class foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >

Definition at line 114 of file ParquetInPlaceEncoder.h.

Constructor & Destructor Documentation

template<typename V, typename T, typename NullType = V>

foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::TypedParquetInPlaceEncoder	(	Data_Namespace::AbstractBuffer *	buffer,
		const ColumnDescriptor *	column_desciptor,
		const parquet::ColumnDescriptor *	parquet_column_descriptor
	)

inline

Definition at line 116 of file ParquetInPlaceEncoder.h.

       : ParquetInPlaceEncoder(
             buffer,
             sizeof(V),
             parquet::GetTypeByteSize(parquet_column_descriptor->physical_type()))
       , current_batch_offset_(0) {
     if (auto detect_buffer = dynamic_cast<TypedParquetDetectBuffer*>(buffer_)) {
       setDetectBufferConverterType();
     }
   }

template<typename V, typename T, typename NullType = V>

foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::TypedParquetInPlaceEncoder	(	Data_Namespace::AbstractBuffer *	buffer,
		const size_t	omnisci_data_type_byte_size,
		const size_t	parquet_data_type_byte_size
	)

inline

Definition at line 129 of file ParquetInPlaceEncoder.h.

       : ParquetInPlaceEncoder(buffer, sizeof(V), parquet_data_type_byte_size)
       , current_batch_offset_(0) {
     if (auto detect_buffer = dynamic_cast<TypedParquetDetectBuffer*>(buffer_)) {
       setDetectBufferConverterType();
     }
   }

Member Function Documentation

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

This is a specialization of ParquetInPlaceEncoder::appendData for known types that allows for optimization.

See comment for ParquetInPlaceEncoder::appendData for details.

Reimplemented from foreign_storage::ParquetInPlaceEncoder.

Definition at line 264 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::ParquetStringEncoder< V >::appendData(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendDataTrackErrors(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::validateAndAppendData().

                                            {
     if (std::is_same<V, T>::value && values_read == levels_read) {
       if (!encodingIsIdentityForSameTypes()) {
         for (int64_t i = 0; i < levels_read; ++i) {
           encodeAndCopy(values + i * omnisci_data_type_byte_size_,
                         values + i * omnisci_data_type_byte_size_);
         }
       }
       buffer_->append(values, levels_read * omnisci_data_type_byte_size_);
     } else {
       ParquetInPlaceEncoder::appendData(
           def_levels, rep_levels, values_read, levels_read, values);
     }
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 191 of file ParquetInPlaceEncoder.h.

                                                       {
     CHECK(is_error_tracking_enabled_);
     int64_t i, j;
     for (i = 0, j = 0; i < levels_read; ++i) {
       if (def_levels[i]) {
         try {
           CHECK(j < values_read);
           validateUsingEncodersColumnType(values, j++);
         } catch (const std::runtime_error& error) {
           invalid_indices_.insert(current_chunk_offset_ + i);
         }
       } else if (column_type_.get_notnull()) {  // item is null for NOT NULL column
         invalid_indices_.insert(current_chunk_offset_ + i);
       }
     }
     current_chunk_offset_ += levels_read;
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::copy	(	const int8_t *	omnisci_data_bytes_source,
		int8_t *	omnisci_data_bytes_destination
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 299 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::ParquetStringEncoder< V >::encodeAndCopy().

                                                              {
     const auto& omnisci_data_value_source =
         reinterpret_cast<const V*>(omnisci_data_bytes_source)[0];
     auto& omnisci_data_value_destination =
         reinterpret_cast<V*>(omnisci_data_bytes_destination)[0];
     omnisci_data_value_destination = omnisci_data_value_source;
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

std::string foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::elementToString ( const V & element ) const

inline

Definition at line 156 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::encodedDataToString(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::setDetectBufferConverterType().

                                                     {
     // handle specialized cases that require specific formating when converting to string
     auto null_value = get_null_value<NullType>();
     if (element == null_value) {
       return "NULL";
     }
     if (isIntegralType(ParquetEncoder::column_type_)) {
       return integralTypeToString(element);
     }
     return std::to_string(element);
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::encodeAndCopyContiguous	(	const int8_t *	parquet_data_bytes,
		int8_t *	omnisci_data_bytes,
		const size_t	num_elements
	)

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 283 of file ParquetInPlaceEncoder.h.

                                                                    {
     auto parquet_data_ptr = reinterpret_cast<const T*>(parquet_data_bytes);
     auto omnisci_data_ptr = reinterpret_cast<V*>(omnisci_data_bytes);
     for (size_t i = 0; i < num_elements; ++i) {
       encodeAndCopy(reinterpret_cast<const int8_t*>(&parquet_data_ptr[i]),
                     reinterpret_cast<int8_t*>(&omnisci_data_ptr[i]));
     }
   }

template<typename V, typename T, typename NullType = V>

std::string foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::encodedDataToString ( const int8_t * bytes ) const

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 168 of file ParquetInPlaceEncoder.h.

                                                                     {
     const auto& element = reinterpret_cast<const V*>(bytes)[0];
     return elementToString(element);
   }

template<typename V, typename T, typename NullType = V>

virtual bool foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::encodingIsIdentityForSameTypes ( ) const

inlineprotectedvirtual

Reimplemented in foreign_storage::ParquetStringEncoder< V >, foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >, and foreign_storage::ParquetDecimalEncoder< V, T, NullType >.

Definition at line 362 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendData().

362 { return false; }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::eraseInvalidIndicesInBuffer ( const InvalidRowGroupIndices & invalid_indices )

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 239 of file ParquetInPlaceEncoder.h.

                                                               {
     if (invalid_indices.empty()) {
       return;
     }
     auto omnisci_data_values = reinterpret_cast<V*>(buffer_->getMemoryPtr());
     CHECK(buffer_->size() % omnisci_data_type_byte_size_ == 0);
     size_t num_elements = buffer_->size() / omnisci_data_type_byte_size_;
     std::remove_if(
         omnisci_data_values, omnisci_data_values + num_elements, [&](const V& value) {
           const V* start = omnisci_data_values;
           auto index = std::distance(start, &value);
           return invalid_indices.find(index) != invalid_indices.end();
         });
     size_t num_bytes_erased = invalid_indices.size() * omnisci_data_type_byte_size_;
     CHECK(num_bytes_erased <= buffer_->size());
     buffer_->setSize(buffer_->size() - num_bytes_erased);
   }

template<typename V, typename T, typename NullType = V>

std::pair<V, V> foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::getEncodedStats	(	const parquet::ColumnDescriptor *	parquet_column_descriptor,
		std::shared_ptr< parquet::Statistics >	stats
	)

inlineprivate

Definition at line 394 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata().

                                               {
     V stats_min, stats_max;
     auto min_string = stats->EncodeMin();
     auto max_string = stats->EncodeMax();
     if constexpr (std::is_same<T, parquet::FixedLenByteArray>::value) {
       CHECK_EQ(parquet_column_descriptor->physical_type(),
                parquet::Type::FIXED_LEN_BYTE_ARRAY);
       parquet::FixedLenByteArray min_byte_array, max_byte_array;
       min_byte_array.ptr = reinterpret_cast<const uint8_t*>(min_string.data());
       max_byte_array.ptr = reinterpret_cast<const uint8_t*>(max_string.data());
       encodeAndCopy(reinterpret_cast<int8_t*>(&min_byte_array),
                     reinterpret_cast<int8_t*>(&stats_min));
       encodeAndCopy(reinterpret_cast<int8_t*>(&max_byte_array),
                     reinterpret_cast<int8_t*>(&stats_max));
     } else if constexpr (std::is_same<T, parquet::ByteArray>::value) {
       CHECK_EQ(parquet_column_descriptor->physical_type(), parquet::Type::BYTE_ARRAY);
       parquet::ByteArray min_byte_array, max_byte_array;
       min_byte_array.ptr = reinterpret_cast<const uint8_t*>(min_string.data());
       min_byte_array.len = min_string.length();
       max_byte_array.ptr = reinterpret_cast<const uint8_t*>(max_string.data());
       max_byte_array.len = max_string.length();
       encodeAndCopy(reinterpret_cast<int8_t*>(&min_byte_array),
                     reinterpret_cast<int8_t*>(&stats_min));
       encodeAndCopy(reinterpret_cast<int8_t*>(&max_byte_array),
                     reinterpret_cast<int8_t*>(&stats_max));
     } else {
       encodeAndCopy(reinterpret_cast<int8_t*>(min_string.data()),
                     reinterpret_cast<int8_t*>(&stats_min));
       encodeAndCopy(reinterpret_cast<int8_t*>(max_string.data()),
                     reinterpret_cast<int8_t*>(&stats_max));
     }
     return {stats_min, stats_max};
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

std::shared_ptr<ChunkMetadata> foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::getRowGroupMetadata	(	const parquet::RowGroupMetaData *	group_metadata,
		const int	parquet_column_index,
		const SQLTypeInfo &	column_type
	)

inlineoverridevirtual

Reimplemented from foreign_storage::ParquetEncoder.

Definition at line 308 of file ParquetInPlaceEncoder.h.

                                                {
     auto metadata = ParquetEncoder::createMetadata(column_type);
     auto column_metadata = group_metadata->ColumnChunk(parquet_column_index);
 
     // update statistics
     auto parquet_column_descriptor =
         group_metadata->schema()->Column(parquet_column_index);
 
     if (ParquetEncoder::validate_metadata_stats_) {
       auto stats = validate_and_get_column_metadata_statistics(column_metadata.get());
       if (stats->HasMinMax()) {
         // validate statistics if validation applicable as part of encoding
         if (auto parquet_scalar_validator =
                 dynamic_cast<ParquetMetadataValidator*>(this)) {
           try {
             parquet_scalar_validator->validate(
                 stats,
                 column_type.is_array() ? column_type.get_elem_type() : column_type);
           } catch (const std::exception& e) {
             std::stringstream error_message;
             error_message
                 << e.what() << " Error validating statistics of Parquet column '"
                 << group_metadata->schema()->Column(parquet_column_index)->name() << "'";
             throw std::runtime_error(error_message.str());
           }
         }
 
         auto [stats_min, stats_max] = getEncodedStats(parquet_column_descriptor, stats);
         auto updated_chunk_stats = getUpdatedStats(stats_min, stats_max, column_type);
         metadata->fillChunkStats(updated_chunk_stats.min,
                                  updated_chunk_stats.max,
                                  metadata->chunkStats.has_nulls);
       }
       auto null_count = stats->null_count();
       validateNullCount(group_metadata->schema()->Column(parquet_column_index)->name(),
                         null_count,
                         column_type);
       metadata->chunkStats.has_nulls = null_count > 0;
     }
 
     // update sizing
     metadata->numBytes =
         sizeof(NullType)  // use NullType byte size since it is guaranteed to
                           // be the byte size of stored data
         * column_metadata->num_values();
     metadata->numElements = group_metadata->num_rows();
 
     return metadata;
   }

template<typename V, typename T, typename NullType = V>

std::pair<T, T> foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::getUnencodedStats ( std::shared_ptr< parquet::Statistics > stats ) const

inlineprotected

Definition at line 364 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::ParquetDateInSecondsEncoder< NullType >::validate(), foreign_storage::ParquetTimestampEncoder< V, T, conversion_denominator *kSecsPerDay, NullType >::validate(), foreign_storage::ParquetUnsignedFixedLengthEncoder< V, T, U, NullType >::validate(), and foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointMetadata().

                                                                                 {
     T stats_min = reinterpret_cast<T*>(stats->EncodeMin().data())[0];
     T stats_max = reinterpret_cast<T*>(stats->EncodeMax().data())[0];
     return {stats_min, stats_max};
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

static ChunkStats foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::getUpdatedStats	(	V &	stats_min,
		V &	stats_max,
		const SQLTypeInfo &	column_type
	)

inlinestaticprivate

Definition at line 371 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata().

                                                                     {
     ForeignStorageBuffer buffer;
     buffer.initEncoder(column_type);
     auto encoder = buffer.getEncoder();
 
     if (column_type.is_array()) {
       ArrayDatum min_datum(
           sizeof(V), reinterpret_cast<int8_t*>(&stats_min), false, DoNothingDeleter());
       ArrayDatum max_datum(
           sizeof(V), reinterpret_cast<int8_t*>(&stats_max), false, DoNothingDeleter());
       std::vector<ArrayDatum> min_max_datums{min_datum, max_datum};
       encoder->updateStats(&min_max_datums, 0, 1);
     } else {
       encoder->updateStats(reinterpret_cast<int8_t*>(&stats_min), 1);
       encoder->updateStats(reinterpret_cast<int8_t*>(&stats_max), 1);
     }
     auto updated_chunk_stats_metadata = std::make_shared<ChunkMetadata>();
     encoder->getMetadata(updated_chunk_stats_metadata);
     return updated_chunk_stats_metadata->chunkStats;
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

std::string foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::integralTypeToString ( const V & element ) const

inline

Definition at line 145 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::elementToString().

                                                          {
     Datum d;
     d.bigintval = element;
     return DatumToString(d, ParquetEncoder::column_type_);
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

bool foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::isIntegralType ( const SQLTypeInfo & type ) const

inline

Definition at line 151 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::elementToString().

                                                      {
     return type.is_timestamp() || type.is_time() || type.is_date() || type.is_boolean() ||
            type.is_decimal() || type.is_integer();
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::reserve ( const size_t num_append_elements )

inlineoverridevirtual

Implements foreign_storage::ParquetInPlaceEncoder.

Definition at line 187 of file ParquetInPlaceEncoder.h.

                                                           {
     buffer_->reserve(buffer_->size() + (num_append_elements * sizeof(V)));
   }

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::setDetectBufferConverterType ( )

inline

Definition at line 173 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::TypedParquetInPlaceEncoder().

                                       {
     auto detect_buffer = dynamic_cast<TypedParquetDetectBuffer*>(buffer_);
     CHECK(detect_buffer);
     std::function<std::string(const V&)> element_to_string = [this](const V& element) {
       return this->elementToString(element);
     };
     detect_buffer->setConverterType<V>(element_to_string);
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::setNull ( int8_t * omnisci_data_bytes )

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 294 of file ParquetInPlaceEncoder.h.

                                                     {
     auto& omnisci_data_value = reinterpret_cast<V*>(omnisci_data_bytes)[0];
     omnisci_data_value = get_null_value<NullType>();
   }

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::validate	(	const int8_t *	parquet_data,
		const int64_t	j,
		const SQLTypeInfo &	column_type
	)		const

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Reimplemented in foreign_storage::ParquetTimestampEncoder< V, T, conversion_denominator, NullType >, and foreign_storage::ParquetTimestampEncoder< V, T, conversion_denominator *kSecsPerDay, NullType >.

Definition at line 139 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::validateAndAppendData(), and foreign_storage::TypedParquetInPlaceEncoder< V, V >::validateUsingEncodersColumnType().

                                                                {
     // no-op by default
   }

Here is the caller graph for this function:

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::validateAndAppendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values,
		const SQLTypeInfo &	column_type,
		InvalidRowGroupIndices &	invalid_indices
	)

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 217 of file ParquetInPlaceEncoder.h.

                                                                                {
     int64_t i, j;
     for (i = 0, j = 0; i < levels_read; ++i) {
       if (def_levels[i]) {
         try {
           CHECK(j < values_read);
           validate(values, j++, column_type);
         } catch (const std::runtime_error& error) {
           invalid_indices.insert(current_batch_offset_ + i);
         }
       }
     }
     current_batch_offset_ += levels_read;
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

template<typename V, typename T, typename NullType = V>

void foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::validateUsingEncodersColumnType	(	const int8_t *	parquet_data,
		const int64_t	j
	)		const

inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 182 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendDataTrackErrors().

                                                                        {
     validate(parquet_data, j, column_type_);
   }

Here is the caller graph for this function:

Member Data Documentation

template<typename V, typename T, typename NullType = V>

int64_t foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::current_batch_offset_ = 0

private

Definition at line 430 of file ParquetInPlaceEncoder.h.

Referenced by foreign_storage::TypedParquetInPlaceEncoder< V, V >::validateAndAppendData().

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetInPlaceEncoder.h

Public Member Functions

Protected Member Functions

Private Member Functions

Static Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

template<typename V, typename T, typename NullType = V> class foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation

template<typename V, typename T, typename NullType = V>
class foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >