#include <ParquetGeospatialEncoder.h>

Inheritance diagram for foreign_storage::ParquetGeospatialEncoder:

Collaboration diagram for foreign_storage::ParquetGeospatialEncoder:

Public Member Functions
	ParquetGeospatialEncoder (const bool geo_validate_geometry)

	ParquetGeospatialEncoder (const parquet::ColumnDescriptor *parquet_column_descriptor, std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const bool geo_validate_geometry)

void	appendData (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

void	appendDataTrackErrors (const int16_t def_levels, const int16_t rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override

Public Member Functions inherited from foreign_storage::ParquetEncoder
	ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)

virtual	~ParquetEncoder ()=default

virtual std::shared_ptr < ChunkMetadata >	getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type)

RejectedRowIndices	getRejectedRowIndices () const

virtual void	disableMetadataStatsValidation ()

virtual void	initializeErrorTracking ()

virtual void	initializeColumnType (const SQLTypeInfo &column_type)

Public Member Functions inherited from foreign_storage::GeospatialEncoder
virtual	~GeospatialEncoder ()=default

	GeospatialEncoder (const bool geo_validate_geometry)

	GeospatialEncoder (std::list< Chunk_NS::Chunk > &chunks, const bool geo_validate_geometry)

	GeospatialEncoder (std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const bool geo_validate_geometry)

Additional Inherited Members
Protected Types inherited from foreign_storage::GeospatialEncoder
enum	GeoColumnType { COORDS, BOUNDS, RING_OR_LINE_SIZES, POLY_RINGS }

Protected Member Functions inherited from foreign_storage::GeospatialEncoder
void	appendBaseDataAndUpdateMetadata (const int64_t row_count)

void	validateChunksSizing (std::list< Chunk_NS::Chunk > &chunks) const

void	validateMetadataSizing (std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata) const

void	appendArrayDatumsToBufferAndUpdateMetadata ()

void	appendToArrayEncoderAndUpdateMetadata (const std::vector< ArrayDatum > &datum_parse_buffer, Encoder encoder, ChunkMetadata chunk_metadata) const

void	processGeoElement (std::string_view geo_string_view)

void	processNullGeoElement ()

void	clearParseBuffers ()

void	clearDatumBuffers ()

template<typename T >
std::list< T >::iterator	getIteratorForGeoColumnType (std::list< T > &list, const SQLTypes column_type, const GeoColumnType geo_column)

std::tuple< Encoder , ChunkMetadata , const ColumnDescriptor * >	initEncoderAndGetEncoderAndMetadata (std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const SQLTypes sql_type, GeoColumnType geo_column_type)

const ColumnDescriptor *	getColumnDescriptor (std::list< Chunk_NS::Chunk > &chunks, const SQLTypes sql_type, GeoColumnType geo_column_type)

bool	hasBoundsColumn () const

bool	hasRingOrLineSizesColumn () const

bool	hasPolyRingsColumn () const

Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr < ChunkMetadata >	createMetadata (const SQLTypeInfo &column_type)

static void	throwNotNullViolation (const std::string &parquet_column_name)

static void	validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)

Static Protected Member Functions inherited from foreign_storage::GeospatialEncoder
static void	throwMalformedGeoElement (const std::string &omnisci_column_name)

static void	throwMismatchedGeoElement (const std::string &omnisci_column_name)

Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer *	buffer_

bool	is_error_tracking_enabled_

RejectedRowIndices	invalid_indices_

size_t	current_chunk_offset_

SQLTypeInfo	column_type_

bool	validate_metadata_stats_

Protected Attributes inherited from foreign_storage::GeospatialEncoder
const ColumnDescriptor *	geo_column_descriptor_

StringNoneEncoder *	base_column_encoder_

Encoder *	coords_column_encoder_

Encoder *	bounds_column_encoder_

Encoder *	ring_or_line_sizes_column_encoder_

Encoder *	poly_rings_column_encoder_

ChunkMetadata *	base_column_metadata_

ChunkMetadata *	coords_column_metadata_

ChunkMetadata *	bounds_column_metadata_

ChunkMetadata *	ring_or_line_sizes_column_metadata_

ChunkMetadata *	poly_rings_column_metadata_

const ColumnDescriptor *	coords_column_descriptor_

const ColumnDescriptor *	bounds_column_descriptor_

const ColumnDescriptor *	ring_or_line_sizes_column_descriptor_

const ColumnDescriptor *	poly_rings_column_descriptor_

std::vector< std::string >	base_values_

std::vector< double >	coords_parse_buffer_

std::vector< double >	bounds_parse_buffer_

std::vector< int >	ring_or_line_sizes_parse_buffer_

std::vector< int >	poly_rings_parse_buffer_

std::vector< ArrayDatum >	coords_datum_buffer_

std::vector< ArrayDatum >	bounds_datum_buffer_

std::vector< ArrayDatum >	ring_or_line_sizes_datum_buffer_

std::vector< ArrayDatum >	poly_rings_datum_buffer_

bool	geo_validate_geometry_

Detailed Description

Definition at line 26 of file ParquetGeospatialEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetGeospatialEncoder::ParquetGeospatialEncoder ( const bool geo_validate_geometry )

inline

Definition at line 28 of file ParquetGeospatialEncoder.h.

29 : ParquetEncoder(nullptr), GeospatialEncoder(geo_validate_geometry) {}

foreign_storage::ParquetEncoder::ParquetEncoder

ParquetEncoder(Data_Namespace::AbstractBuffer *buffer)

Definition: ParquetEncoder.h:32

foreign_storage::GeospatialEncoder::GeospatialEncoder

GeospatialEncoder(const bool geo_validate_geometry)

Definition: GeospatialEncoder.h:43

foreign_storage::ParquetGeospatialEncoder::ParquetGeospatialEncoder	(	const parquet::ColumnDescriptor *	parquet_column_descriptor,
		std::list< Chunk_NS::Chunk > &	chunks,
		std::list< std::unique_ptr< ChunkMetadata >> &	chunk_metadata,
		const bool	geo_validate_geometry
	)

inline

Definition at line 31 of file ParquetGeospatialEncoder.h.

35 : ParquetEncoder(nullptr)

36 , GeospatialEncoder(chunks, chunk_metadata, geo_validate_geometry) {}

foreign_storage::ParquetEncoder::ParquetEncoder

ParquetEncoder(Data_Namespace::AbstractBuffer *buffer)

Definition: ParquetEncoder.h:32

foreign_storage::GeospatialEncoder::GeospatialEncoder

GeospatialEncoder(const bool geo_validate_geometry)

Definition: GeospatialEncoder.h:43

Member Function Documentation

void foreign_storage::ParquetGeospatialEncoder::appendData	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

add null if failed

Implements foreign_storage::ParquetEncoder.

Definition at line 38 of file ParquetGeospatialEncoder.h.

References foreign_storage::GeospatialEncoder::appendArrayDatumsToBufferAndUpdateMetadata(), foreign_storage::GeospatialEncoder::appendBaseDataAndUpdateMetadata(), CHECK, foreign_storage::GeospatialEncoder::clearDatumBuffers(), foreign_storage::GeospatialEncoder::clearParseBuffers(), foreign_storage::ParquetEncoder::column_type_, foreign_storage::ParquetEncoder::current_chunk_offset_, foreign_storage::ParquetEncoder::invalid_indices_, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, foreign_storage::GeospatialEncoder::processGeoElement(), and foreign_storage::GeospatialEncoder::processNullGeoElement().

Referenced by appendDataTrackErrors().

                                            {
     auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
 
     clearDatumBuffers();
 
     for (int64_t i = 0, j = 0; i < levels_read; ++i) {
       clearParseBuffers();
       if (def_levels[i] == 0) {
         if (is_error_tracking_enabled_ &&
             column_type_
                 .get_notnull()) {  // mark as invalid due to a null in a NOT NULL column
           invalid_indices_.insert(current_chunk_offset_ + i);
         }
         processNullGeoElement();
       } else {
         CHECK(j < values_read);
         auto& byte_array = parquet_data_ptr[j++];
         auto geo_string_view = std::string_view{
             reinterpret_cast<const char*>(byte_array.ptr), byte_array.len};
         try {
           processGeoElement(geo_string_view);
         } catch (const std::runtime_error& error) {
           if (is_error_tracking_enabled_) {
             invalid_indices_.insert(current_chunk_offset_ + i);
             clearParseBuffers();
             processNullGeoElement();
           } else {
             throw;
           }
         }
       }
     }
 
     appendArrayDatumsToBufferAndUpdateMetadata();
 
     appendBaseDataAndUpdateMetadata(levels_read);
 
     if (is_error_tracking_enabled_) {
       current_chunk_offset_ += levels_read;
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ParquetGeospatialEncoder::appendDataTrackErrors	(	const int16_t *	def_levels,
		const int16_t *	rep_levels,
		const int64_t	values_read,
		const int64_t	levels_read,
		int8_t *	values
	)

inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 85 of file ParquetGeospatialEncoder.h.

References appendData(), CHECK, and foreign_storage::ParquetEncoder::is_error_tracking_enabled_.

                                                       {
     CHECK(is_error_tracking_enabled_);
     // `appendData` modifies its behaviour based on the
     // `is_error_tracking_enabled_` flag to handle this case
     appendData(def_levels, rep_levels, values_read, levels_read, values);
   }

Here is the call graph for this function:

The documentation for this class was generated from the following file:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ParquetGeospatialEncoder.h

Public Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation