#include <ParquetGeospatialImportEncoder.h>

Inheritance diagram for foreign_storage::ParquetGeospatialImportEncoder:

Collaboration diagram for foreign_storage::ParquetGeospatialImportEncoder:

Public Member Functions
	ParquetGeospatialImportEncoder ()

	ParquetGeospatialImportEncoder (std::list< Chunk_NS::Chunk > &chunks)

void	validateAndAppendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices) override

void	eraseInvalidIndicesInBuffer (const InvalidRowGroupIndices &invalid_indices) override

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

virtual std::shared_ptr < ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type)

RejectedRowIndices	getRejectedRowIndices () const

virtual void	disableMetadataStatsValidation ()

virtual void	initializeErrorTracking (const SQLTypeInfo &column_type)

Public Member Functions inherited from foreign_storage::GeospatialEncoder
virtual	~GeospatialEncoder ()=default

	GeospatialEncoder (const RenderGroupAnalyzerMap *render_group_analyzer_map)

	GeospatialEncoder (std::list< Chunk_NS::Chunk > &chunks, const RenderGroupAnalyzerMap *render_group_analyzer_map)

	GeospatialEncoder (std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const RenderGroupAnalyzerMap *render_group_analyzer_map)

Private Member Functions
void	appendArrayDatumsIfApplicable (TypedParquetStorageBuffer< ArrayDatum > *column_buffer, const std::vector< ArrayDatum > &datum_buffer)

void	appendArrayDatumsToBuffer ()

void	appendBaseAndRenderGroupData (const int64_t row_count)

AbstractBuffer *	getBuffer (std::list< Chunk_NS::Chunk > &chunks, const SQLTypes sql_type, GeoColumnType geo_column_type)

Private Attributes
int64_t	current_batch_offset_

InvalidRowGroupIndices *	invalid_indices_

TypedParquetStorageBuffer < std::string > *	base_column_buffer_

TypedParquetStorageBuffer < ArrayDatum > *	coords_column_buffer_

TypedParquetStorageBuffer < ArrayDatum > *	bounds_column_buffer_

TypedParquetStorageBuffer < ArrayDatum > *	ring_sizes_column_buffer_

TypedParquetStorageBuffer < ArrayDatum > *	poly_rings_column_buffer_

AbstractBuffer *	render_group_column_buffer_

Additional Inherited Members
Protected Types inherited from foreign_storage::GeospatialEncoder
enum	GeoColumnType { COORDS, BOUNDS, RING_SIZES, POLY_RINGS, RENDER_GROUP }

Protected Member Functions inherited from foreign_storage::GeospatialEncoder
void	appendBaseAndRenderGroupDataAndUpdateMetadata (const int64_t row_count)

void	validateChunksSizing (std::list< Chunk_NS::Chunk > &chunks) const

void	validateMetadataSizing (std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata) const

void	appendArrayDatumsToBufferAndUpdateMetadata ()

void	appendToArrayEncoderAndUpdateMetadata (const std::vector< ArrayDatum > &datum_parse_buffer, Encoder encoder, ChunkMetadata chunk_metadata) const

void	processGeoElement (std::string_view geo_string_view)

void	processNullGeoElement ()

void	clearParseBuffers ()

void	clearDatumBuffers ()

template<typename T >
std::list< T >::iterator	getIteratorForGeoColumnType (std::list< T > &list, const SQLTypes column_type, const GeoColumnType geo_column)

std::tuple< Encoder , ChunkMetadata , const ColumnDescriptor * >	initEncoderAndGetEncoderAndMetadata (std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const SQLTypes sql_type, GeoColumnType geo_column_type)

const ColumnDescriptor *	getColumnDescriptor (std::list< Chunk_NS::Chunk > &chunks, const SQLTypes sql_type, GeoColumnType geo_column_type)

bool	hasBoundsColumn () const

bool	hasRingSizesColumn () const

bool	hasRenderGroupColumn () const

bool	hasPolyRingsColumn () const

Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Static Protected Member Functions inherited from foreign_storage::GeospatialEncoder
static void	throwMalformedGeoElement (const std::string &omnisci_column_name)

static void	throwMismatchedGeoElement (const std::string &omnisci_column_name)

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Protected Attributes inherited from foreign_storage::GeospatialEncoder
const ColumnDescriptor *	geo_column_descriptor_

StringNoneEncoder *	base_column_encoder_

Encoder *	coords_column_encoder_

Encoder *	bounds_column_encoder_

Encoder *	ring_sizes_column_encoder_

Encoder *	poly_rings_column_encoder_

Encoder *	render_group_column_encoder_

ChunkMetadata *	base_column_metadata_

ChunkMetadata *	coords_column_metadata_

ChunkMetadata *	bounds_column_metadata_

ChunkMetadata *	ring_sizes_column_metadata_

ChunkMetadata *	poly_rings_column_metadata_

ChunkMetadata *	render_group_column_metadata_

const ColumnDescriptor *	coords_column_descriptor_

const ColumnDescriptor *	bounds_column_descriptor_

const ColumnDescriptor *	ring_sizes_column_descriptor_

const ColumnDescriptor *	poly_rings_column_descriptor_

const ColumnDescriptor *	render_group_column_descriptor_

std::vector< std::string >	base_values_

std::vector< double >	coords_parse_buffer_

std::vector< double >	bounds_parse_buffer_

std::vector< int >	ring_sizes_parse_buffer_

std::vector< int >	poly_rings_parse_buffer_

std::vector< ArrayDatum >	coords_datum_buffer_

std::vector< ArrayDatum >	bounds_datum_buffer_

std::vector< ArrayDatum >	ring_sizes_datum_buffer_

std::vector< ArrayDatum >	poly_rings_datum_buffer_

std::vector< int32_t >	render_group_value_buffer_

const RenderGroupAnalyzerMap *	render_group_analyzer_map_

Static Protected Attributes inherited from foreign_storage::GeospatialEncoder
static constexpr bool	PROMOTE_POLYGON_TO_MULTIPOLYGON = true

Detailed Description

Definition at line 27 of file ParquetGeospatialImportEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetGeospatialImportEncoder::ParquetGeospatialImportEncoder ( )

inline

Definition at line 31 of file ParquetGeospatialImportEncoder.h.

       : ParquetEncoder(nullptr)
       , GeospatialEncoder(nullptr)
       , current_batch_offset_(0)
       , invalid_indices_(nullptr) {}

foreign_storage::ParquetGeospatialImportEncoder::ParquetGeospatialImportEncoder ( std::list< Chunk_NS::Chunk > & chunks )

inline

Definition at line 37 of file ParquetGeospatialImportEncoder.h.

       : ParquetEncoder(nullptr)
       , GeospatialEncoder(chunks, nullptr)
       , current_batch_offset_(0)
       , invalid_indices_(nullptr)
       , base_column_buffer_(nullptr)
       , coords_column_buffer_(nullptr)
       , bounds_column_buffer_(nullptr)
       , ring_sizes_column_buffer_(nullptr)
       , poly_rings_column_buffer_(nullptr)
       , render_group_column_buffer_(nullptr) {
     CHECK(geo_column_descriptor_->columnType.is_geometry());
 
     const auto geo_column_type = geo_column_descriptor_->columnType.get_type();
 
     base_column_buffer_ = dynamic_cast<TypedParquetStorageBuffer<std::string>*>(
         chunks.begin()->getBuffer());
     CHECK(base_column_buffer_);
 
     // initialize coords column
     coords_column_buffer_ = dynamic_cast<TypedParquetStorageBuffer<ArrayDatum>*>(
         getBuffer(chunks, geo_column_type, COORDS));
     CHECK(coords_column_buffer_);
 
     // initialize bounds column
     if (hasBoundsColumn()) {
       bounds_column_buffer_ = dynamic_cast<TypedParquetStorageBuffer<ArrayDatum>*>(
           getBuffer(chunks, geo_column_type, BOUNDS));
       CHECK(bounds_column_buffer_);
     }
 
     // initialize ring sizes column & render group column
     if (hasRingSizesColumn()) {
       ring_sizes_column_buffer_ = dynamic_cast<TypedParquetStorageBuffer<ArrayDatum>*>(
           getBuffer(chunks, geo_column_type, RING_SIZES));
       CHECK(ring_sizes_column_buffer_);
     }
     if (hasRenderGroupColumn()) {
       render_group_column_buffer_ = getBuffer(chunks, geo_column_type, RENDER_GROUP);
       CHECK(render_group_column_buffer_);
     }
 
     // initialize poly rings column
     if (hasPolyRingsColumn()) {
       poly_rings_column_buffer_ = dynamic_cast<TypedParquetStorageBuffer<ArrayDatum>*>(
           getBuffer(chunks, geo_column_type, POLY_RINGS));
       CHECK(poly_rings_column_buffer_);
     }
   }

Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ParquetGeospatialImportEncoder::appendArrayDatumsIfApplicable	(	TypedParquetStorageBuffer< ArrayDatum > *	column_buffer,
		const std::vector< ArrayDatum > &	datum_buffer
	)

inlineprivate

Definition at line 167 of file ParquetGeospatialImportEncoder.h.

References foreign_storage::TypedParquetStorageBuffer< Type >::appendElement(), and CHECK.

Referenced by appendArrayDatumsToBuffer().

                                                                                 {
     if (column_buffer) {
       for (const auto& datum : datum_buffer) {
         column_buffer->appendElement(datum);
       }
     } else {
       CHECK(datum_buffer.empty());
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetGeospatialImportEncoder::appendArrayDatumsToBuffer ( )

inlineprivate

Definition at line 178 of file ParquetGeospatialImportEncoder.h.

References appendArrayDatumsIfApplicable(), bounds_column_buffer_, foreign_storage::GeospatialEncoder::bounds_datum_buffer_, coords_column_buffer_, foreign_storage::GeospatialEncoder::coords_datum_buffer_, poly_rings_column_buffer_, foreign_storage::GeospatialEncoder::poly_rings_datum_buffer_, ring_sizes_column_buffer_, and foreign_storage::GeospatialEncoder::ring_sizes_datum_buffer_.

Referenced by appendData().

                                    {
     appendArrayDatumsIfApplicable(coords_column_buffer_, coords_datum_buffer_);
     appendArrayDatumsIfApplicable(bounds_column_buffer_, bounds_datum_buffer_);
     appendArrayDatumsIfApplicable(ring_sizes_column_buffer_, ring_sizes_datum_buffer_);
     appendArrayDatumsIfApplicable(poly_rings_column_buffer_, poly_rings_datum_buffer_);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetGeospatialImportEncoder::appendBaseAndRenderGroupData ( const int64_t row_count )

inlineprivate

Definition at line 185 of file ParquetGeospatialImportEncoder.h.

References Data_Namespace::AbstractBuffer::append(), foreign_storage::TypedParquetStorageBuffer< Type >::appendElement(), base_column_buffer_, render_group_column_buffer_, and foreign_storage::GeospatialEncoder::render_group_value_buffer_.

Referenced by appendData().

                                                              {
     for (int64_t i = 0; i < row_count; ++i) {
       base_column_buffer_->appendElement("");
     }
     if (render_group_column_buffer_) {
       auto data_ptr = reinterpret_cast<int8_t*>(render_group_value_buffer_.data());
       render_group_column_buffer_->append(data_ptr, sizeof(int32_t) * row_count);
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetGeospatialImportEncoder::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

add null if failed

Implements foreign_storage::ParquetEncoder.

Definition at line 121 of file ParquetGeospatialImportEncoder.h.

References appendArrayDatumsToBuffer(), appendBaseAndRenderGroupData(), CHECK, foreign_storage::GeospatialEncoder::clearDatumBuffers(), foreign_storage::GeospatialEncoder::clearParseBuffers(), current_batch_offset_, invalid_indices_, foreign_storage::GeospatialEncoder::processGeoElement(), and foreign_storage::GeospatialEncoder::processNullGeoElement().

Referenced by validateAndAppendData().

                                            {
     auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
 
     clearDatumBuffers();
 
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       clearParseBuffers();
       if (def_levels[i] == 0) {
         processNullGeoElement();
       } else {
         CHECK(j < values_read);
         auto& byte_array = parquet_data_ptr[j++];
         auto geo_string_view = std::string_view{
             reinterpret_cast<const char*>(byte_array.ptr), byte_array.len};
         try {
           processGeoElement(geo_string_view);
         } catch (const std::runtime_error& error) {
           CHECK(invalid_indices_);
           invalid_indices_->insert(current_batch_offset_ + i);
           clearParseBuffers();
           processNullGeoElement();
         }
       }
     }
 
     appendArrayDatumsToBuffer();
 
     appendBaseAndRenderGroupData(levels_read);
 
     current_batch_offset_ += levels_read;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetGeospatialImportEncoder::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 158 of file ParquetGeospatialImportEncoder.h.

References UNREACHABLE.

                                                       {
     UNREACHABLE() << "unexpected call to appendDataTrackErrors from unsupported encoder";
   }

void foreign_storage::ParquetGeospatialImportEncoder::eraseInvalidIndicesInBuffer ( const InvalidRowGroupIndices & invalid_indices )

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 98 of file ParquetGeospatialImportEncoder.h.

References base_column_buffer_, bounds_column_buffer_, coords_column_buffer_, foreign_storage::TypedParquetStorageBuffer< Type >::eraseInvalidData(), foreign_storage::GeospatialEncoder::hasBoundsColumn(), foreign_storage::GeospatialEncoder::hasPolyRingsColumn(), foreign_storage::GeospatialEncoder::hasRenderGroupColumn(), foreign_storage::GeospatialEncoder::hasRingSizesColumn(), poly_rings_column_buffer_, render_group_column_buffer_, ring_sizes_column_buffer_, Data_Namespace::AbstractBuffer::setSize(), and Data_Namespace::AbstractBuffer::size().

                                                               {
     if (invalid_indices.empty()) {
       return;
     }
     base_column_buffer_->eraseInvalidData(invalid_indices);
     coords_column_buffer_->eraseInvalidData(invalid_indices);
     if (hasBoundsColumn()) {
       bounds_column_buffer_->eraseInvalidData(invalid_indices);
     }
     if (hasRingSizesColumn()) {
       ring_sizes_column_buffer_->eraseInvalidData(invalid_indices);
     }
     if (hasPolyRingsColumn()) {
       poly_rings_column_buffer_->eraseInvalidData(invalid_indices);
     }
     if (hasRenderGroupColumn()) {
       render_group_column_buffer_->setSize(
           sizeof(int32_t) *
           (render_group_column_buffer_->size() - invalid_indices.size()));
     }
   }

Here is the call graph for this function:

AbstractBuffer* foreign_storage::ParquetGeospatialImportEncoder::getBuffer	(	std::list< Chunk_NS::Chunk > &	chunks,
		const SQLTypes	sql_type,
		GeoColumnType	geo_column_type
	)

inlineprivate

Definition at line 195 of file ParquetGeospatialImportEncoder.h.

References foreign_storage::GeospatialEncoder::getIteratorForGeoColumnType().

Referenced by ParquetGeospatialImportEncoder().

                                                            {
     auto chunk = getIteratorForGeoColumnType(chunks, sql_type, geo_column_type);
     auto buffer = chunk->getBuffer();
     return buffer;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetGeospatialImportEncoder::validateAndAppendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values,
		const SQLTypeInfo &	column_type,
		InvalidRowGroupIndices &	invalid_indices
	)

inlineoverridevirtual

Implements foreign_storage::ParquetImportEncoder.

Definition at line 87 of file ParquetGeospatialImportEncoder.h.

References appendData(), and invalid_indices_.

                                                                                {
     invalid_indices_ = &invalid_indices;  // used in assembly algorithm
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

Here is the call graph for this function:

Member Data Documentation

TypedParquetStorageBuffer<std::string>* foreign_storage::ParquetGeospatialImportEncoder::base_column_buffer_

private

Definition at line 205 of file ParquetGeospatialImportEncoder.h.

Referenced by appendBaseAndRenderGroupData(), eraseInvalidIndicesInBuffer(), and ParquetGeospatialImportEncoder().

TypedParquetStorageBuffer<ArrayDatum>* foreign_storage::ParquetGeospatialImportEncoder::bounds_column_buffer_

private

Definition at line 207 of file ParquetGeospatialImportEncoder.h.

Referenced by appendArrayDatumsToBuffer(), eraseInvalidIndicesInBuffer(), and ParquetGeospatialImportEncoder().

TypedParquetStorageBuffer<ArrayDatum>* foreign_storage::ParquetGeospatialImportEncoder::coords_column_buffer_

private

Definition at line 206 of file ParquetGeospatialImportEncoder.h.

Referenced by appendArrayDatumsToBuffer(), eraseInvalidIndicesInBuffer(), and ParquetGeospatialImportEncoder().

int64_t foreign_storage::ParquetGeospatialImportEncoder::current_batch_offset_

private

Definition at line 203 of file ParquetGeospatialImportEncoder.h.

Referenced by appendData().

InvalidRowGroupIndices* foreign_storage::ParquetGeospatialImportEncoder::invalid_indices_

private

Definition at line 204 of file ParquetGeospatialImportEncoder.h.

Referenced by appendData(), and validateAndAppendData().

TypedParquetStorageBuffer<ArrayDatum>* foreign_storage::ParquetGeospatialImportEncoder::poly_rings_column_buffer_

private

Definition at line 209 of file ParquetGeospatialImportEncoder.h.

Referenced by appendArrayDatumsToBuffer(), eraseInvalidIndicesInBuffer(), and ParquetGeospatialImportEncoder().

AbstractBuffer* foreign_storage::ParquetGeospatialImportEncoder::render_group_column_buffer_

private

Definition at line 210 of file ParquetGeospatialImportEncoder.h.

Referenced by appendBaseAndRenderGroupData(), eraseInvalidIndicesInBuffer(), and ParquetGeospatialImportEncoder().

TypedParquetStorageBuffer<ArrayDatum>* foreign_storage::ParquetGeospatialImportEncoder::ring_sizes_column_buffer_

private

Definition at line 208 of file ParquetGeospatialImportEncoder.h.

Referenced by appendArrayDatumsToBuffer(), eraseInvalidIndicesInBuffer(), and ParquetGeospatialImportEncoder().

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetGeospatialImportEncoder.h

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation