#include <ParquetStringNoneEncoder.h>

Inheritance diagram for foreign_storage::ParquetStringNoneEncoder:

Collaboration diagram for foreign_storage::ParquetStringNoneEncoder:

Public Member Functions
	ParquetStringNoneEncoder (Data_Namespace::AbstractBuffer buffer, Data_Namespace::AbstractBuffer index_buffer)

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

virtual std::shared_ptr < ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type)

RejectedRowIndices	getRejectedRowIndices () const

virtual void	disableMetadataStatsValidation ()

virtual void	initializeErrorTracking (const SQLTypeInfo &column_type)

Private Member Functions
void	writeInitialOffsetIfApplicable ()

Private Attributes
Data_Namespace::AbstractBuffer *	index_buffer_

std::vector< int8_t >	encode_buffer_

Additional Inherited Members
Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Detailed Description

Definition at line 27 of file ParquetStringNoneEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetStringNoneEncoder::ParquetStringNoneEncoder	(	Data_Namespace::AbstractBuffer *	buffer,
		Data_Namespace::AbstractBuffer *	index_buffer
	)

inline

Definition at line 29 of file ParquetStringNoneEncoder.h.

       : ParquetEncoder(buffer)
       , index_buffer_(index_buffer)
       , encode_buffer_(LazyParquetChunkLoader::batch_reader_num_elements *
                        sizeof(StringOffsetT)) {}

Member Function Documentation

void foreign_storage::ParquetStringNoneEncoder::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 36 of file ParquetStringNoneEncoder.h.

References Data_Namespace::AbstractBuffer::append(), foreign_storage::ParquetEncoder::buffer_, CHECK, foreign_storage::ParquetEncoder::column_type_, foreign_storage::ParquetEncoder::current_chunk_offset_, encode_buffer_, index_buffer_, foreign_storage::ParquetEncoder::invalid_indices_, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, StringDictionary::MAX_STRLEN, Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::size(), and writeInitialOffsetIfApplicable().

Referenced by appendDataTrackErrors().

                                            {
     CHECK(levels_read > 0);
     writeInitialOffsetIfApplicable();
 
     auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
     auto offsets = reinterpret_cast<StringOffsetT*>(encode_buffer_.data());
     auto last_offset = buffer_->size();
 
     size_t total_len = 0;
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       if (def_levels[i]) {
         CHECK(j < values_read);
         auto& byte_array = parquet_data_ptr[j++];
         if (is_error_tracking_enabled_ && byte_array.len > StringDictionary::MAX_STRLEN) {
           // no-op, or effectively inserting a null: total_len += 0;
         } else {
           total_len += byte_array.len;
         }
       }
       offsets[i] = last_offset + total_len;
     }
     index_buffer_->append(encode_buffer_.data(), levels_read * sizeof(StringOffsetT));
 
     encode_buffer_.resize(std::max<size_t>(total_len, encode_buffer_.size()));
     buffer_->reserve(buffer_->size() + total_len);
     total_len = 0;
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       if (def_levels[i]) {
         CHECK(j < values_read);
         auto& byte_array = parquet_data_ptr[j++];
         if (is_error_tracking_enabled_ && byte_array.len > StringDictionary::MAX_STRLEN) {
           ParquetEncoder::invalid_indices_.insert(ParquetEncoder::current_chunk_offset_ +
                                                   i);
         } else {
           memcpy(encode_buffer_.data() + total_len, byte_array.ptr, byte_array.len);
           total_len += byte_array.len;
         }
       } else if (is_error_tracking_enabled_ &&
                  ParquetEncoder::column_type_
                      .get_notnull()) {  // item is null for NOT NULL column
         ParquetEncoder::invalid_indices_.insert(ParquetEncoder::current_chunk_offset_ +
                                                 i);
       }
     }
     if (is_error_tracking_enabled_) {
       ParquetEncoder::current_chunk_offset_ += levels_read;
     }
     buffer_->append(encode_buffer_.data(), total_len);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetStringNoneEncoder::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 90 of file ParquetStringNoneEncoder.h.

References appendData(), CHECK, and foreign_storage::ParquetEncoder::is_error_tracking_enabled_.

                                                       {
     CHECK(is_error_tracking_enabled_);
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

Here is the call graph for this function:

void foreign_storage::ParquetStringNoneEncoder::writeInitialOffsetIfApplicable ( )

inlineprivate

Definition at line 100 of file ParquetStringNoneEncoder.h.

References Data_Namespace::AbstractBuffer::append(), index_buffer_, and Data_Namespace::AbstractBuffer::size().

Referenced by appendData().

                                         {
     if (!index_buffer_->size()) {
       // write the initial starting offset
       StringOffsetT zero = 0;
       index_buffer_->append(reinterpret_cast<int8_t*>(&zero), sizeof(StringOffsetT));
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

std::vector<int8_t> foreign_storage::ParquetStringNoneEncoder::encode_buffer_

private

Definition at line 109 of file ParquetStringNoneEncoder.h.

Referenced by appendData().

Data_Namespace::AbstractBuffer* foreign_storage::ParquetStringNoneEncoder::index_buffer_

private

Definition at line 108 of file ParquetStringNoneEncoder.h.

Referenced by appendData(), and writeInitialOffsetIfApplicable().

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetStringNoneEncoder.h

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation