OmniSciDB  a987f07e93
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::ParquetGeospatialEncoder Class Reference

#include <ParquetGeospatialEncoder.h>

+ Inheritance diagram for foreign_storage::ParquetGeospatialEncoder:
+ Collaboration diagram for foreign_storage::ParquetGeospatialEncoder:

Public Member Functions

 ParquetGeospatialEncoder (const RenderGroupAnalyzerMap *render_group_analyzer_map)
 
 ParquetGeospatialEncoder (const parquet::ColumnDescriptor *parquet_column_descriptor, std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const RenderGroupAnalyzerMap *render_group_analyzer_map)
 
void appendData (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override
 
void appendDataTrackErrors (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override
 
- Public Member Functions inherited from foreign_storage::ParquetEncoder
 ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~ParquetEncoder ()=default
 
virtual std::shared_ptr
< ChunkMetadata
getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type)
 
RejectedRowIndices getRejectedRowIndices () const
 
virtual void disableMetadataStatsValidation ()
 
virtual void initializeErrorTracking (const SQLTypeInfo &column_type)
 
- Public Member Functions inherited from foreign_storage::GeospatialEncoder
virtual ~GeospatialEncoder ()=default
 
 GeospatialEncoder (const RenderGroupAnalyzerMap *render_group_analyzer_map)
 
 GeospatialEncoder (std::list< Chunk_NS::Chunk > &chunks, const RenderGroupAnalyzerMap *render_group_analyzer_map)
 
 GeospatialEncoder (std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const RenderGroupAnalyzerMap *render_group_analyzer_map)
 

Additional Inherited Members

- Protected Types inherited from foreign_storage::GeospatialEncoder
enum  GeoColumnType {
  COORDS, BOUNDS, RING_OR_LINE_SIZES, POLY_RINGS,
  RENDER_GROUP
}
 
- Protected Member Functions inherited from foreign_storage::GeospatialEncoder
void appendBaseAndRenderGroupDataAndUpdateMetadata (const int64_t row_count)
 
void validateChunksSizing (std::list< Chunk_NS::Chunk > &chunks) const
 
void validateMetadataSizing (std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata) const
 
void appendArrayDatumsToBufferAndUpdateMetadata ()
 
void appendToArrayEncoderAndUpdateMetadata (const std::vector< ArrayDatum > &datum_parse_buffer, Encoder *encoder, ChunkMetadata *chunk_metadata) const
 
void processGeoElement (std::string_view geo_string_view)
 
void processNullGeoElement ()
 
void clearParseBuffers ()
 
void clearDatumBuffers ()
 
template<typename T >
std::list< T >::iterator getIteratorForGeoColumnType (std::list< T > &list, const SQLTypes column_type, const GeoColumnType geo_column)
 
std::tuple< Encoder
*, ChunkMetadata *, const
ColumnDescriptor * > 
initEncoderAndGetEncoderAndMetadata (std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const SQLTypes sql_type, GeoColumnType geo_column_type)
 
const ColumnDescriptorgetColumnDescriptor (std::list< Chunk_NS::Chunk > &chunks, const SQLTypes sql_type, GeoColumnType geo_column_type)
 
bool hasBoundsColumn () const
 
bool hasRingOrLineSizesColumn () const
 
bool hasRenderGroupColumn () const
 
bool hasPolyRingsColumn () const
 
- Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr
< ChunkMetadata
createMetadata (const SQLTypeInfo &column_type)
 
static void throwNotNullViolation (const std::string &parquet_column_name)
 
static void validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)
 
- Static Protected Member Functions inherited from foreign_storage::GeospatialEncoder
static void throwMalformedGeoElement (const std::string &omnisci_column_name)
 
static void throwMismatchedGeoElement (const std::string &omnisci_column_name)
 
- Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBufferbuffer_
 
bool is_error_tracking_enabled_
 
RejectedRowIndices invalid_indices_
 
size_t current_chunk_offset_
 
SQLTypeInfo column_type_
 
bool validate_metadata_stats_
 
- Protected Attributes inherited from foreign_storage::GeospatialEncoder
const ColumnDescriptorgeo_column_descriptor_
 
StringNoneEncoderbase_column_encoder_
 
Encodercoords_column_encoder_
 
Encoderbounds_column_encoder_
 
Encoderring_or_line_sizes_column_encoder_
 
Encoderpoly_rings_column_encoder_
 
Encoderrender_group_column_encoder_
 
ChunkMetadatabase_column_metadata_
 
ChunkMetadatacoords_column_metadata_
 
ChunkMetadatabounds_column_metadata_
 
ChunkMetadataring_or_line_sizes_column_metadata_
 
ChunkMetadatapoly_rings_column_metadata_
 
ChunkMetadatarender_group_column_metadata_
 
const ColumnDescriptorcoords_column_descriptor_
 
const ColumnDescriptorbounds_column_descriptor_
 
const ColumnDescriptorring_or_line_sizes_column_descriptor_
 
const ColumnDescriptorpoly_rings_column_descriptor_
 
const ColumnDescriptorrender_group_column_descriptor_
 
std::vector< std::string > base_values_
 
std::vector< double > coords_parse_buffer_
 
std::vector< double > bounds_parse_buffer_
 
std::vector< int > ring_or_line_sizes_parse_buffer_
 
std::vector< int > poly_rings_parse_buffer_
 
std::vector< ArrayDatumcoords_datum_buffer_
 
std::vector< ArrayDatumbounds_datum_buffer_
 
std::vector< ArrayDatumring_or_line_sizes_datum_buffer_
 
std::vector< ArrayDatumpoly_rings_datum_buffer_
 
std::vector< int32_t > render_group_value_buffer_
 
const RenderGroupAnalyzerMaprender_group_analyzer_map_
 
- Static Protected Attributes inherited from foreign_storage::GeospatialEncoder
static constexpr bool PROMOTE_POLYGON_TO_MULTIPOLYGON = true
 

Detailed Description

Definition at line 26 of file ParquetGeospatialEncoder.h.

Constructor & Destructor Documentation

foreign_storage::ParquetGeospatialEncoder::ParquetGeospatialEncoder ( const RenderGroupAnalyzerMap render_group_analyzer_map)
inline

Definition at line 28 of file ParquetGeospatialEncoder.h.

29  : ParquetEncoder(nullptr), GeospatialEncoder(render_group_analyzer_map) {}
ParquetEncoder(Data_Namespace::AbstractBuffer *buffer)
GeospatialEncoder(const RenderGroupAnalyzerMap *render_group_analyzer_map)
foreign_storage::ParquetGeospatialEncoder::ParquetGeospatialEncoder ( const parquet::ColumnDescriptor *  parquet_column_descriptor,
std::list< Chunk_NS::Chunk > &  chunks,
std::list< std::unique_ptr< ChunkMetadata >> &  chunk_metadata,
const RenderGroupAnalyzerMap render_group_analyzer_map 
)
inline

Definition at line 31 of file ParquetGeospatialEncoder.h.

35  : ParquetEncoder(nullptr)
36  , GeospatialEncoder(chunks, chunk_metadata, render_group_analyzer_map) {}
ParquetEncoder(Data_Namespace::AbstractBuffer *buffer)
GeospatialEncoder(const RenderGroupAnalyzerMap *render_group_analyzer_map)

Member Function Documentation

void foreign_storage::ParquetGeospatialEncoder::appendData ( const int16_t *  def_levels,
const int16_t *  rep_levels,
const int64_t  values_read,
const int64_t  levels_read,
int8_t *  values 
)
inlineoverridevirtual

add null if failed

Implements foreign_storage::ParquetEncoder.

Definition at line 38 of file ParquetGeospatialEncoder.h.

References foreign_storage::GeospatialEncoder::appendArrayDatumsToBufferAndUpdateMetadata(), foreign_storage::GeospatialEncoder::appendBaseAndRenderGroupDataAndUpdateMetadata(), CHECK, foreign_storage::GeospatialEncoder::clearDatumBuffers(), foreign_storage::GeospatialEncoder::clearParseBuffers(), foreign_storage::ParquetEncoder::column_type_, foreign_storage::ParquetEncoder::current_chunk_offset_, foreign_storage::ParquetEncoder::invalid_indices_, foreign_storage::ParquetEncoder::is_error_tracking_enabled_, foreign_storage::GeospatialEncoder::processGeoElement(), and foreign_storage::GeospatialEncoder::processNullGeoElement().

Referenced by appendDataTrackErrors().

42  {
43  auto parquet_data_ptr = reinterpret_cast<const parquet::ByteArray*>(values);
44 
46 
47  for (int64_t i = 0, j = 0; i < levels_read; ++i) {
49  if (def_levels[i] == 0) {
52  .get_notnull()) { // mark as invalid due to a null in a NOT NULL column
54  }
56  } else {
57  CHECK(j < values_read);
58  auto& byte_array = parquet_data_ptr[j++];
59  auto geo_string_view = std::string_view{
60  reinterpret_cast<const char*>(byte_array.ptr), byte_array.len};
61  try {
62  processGeoElement(geo_string_view);
63  } catch (const std::runtime_error& error) {
69  } else {
70  throw;
71  }
72  }
73  }
74  }
75 
77 
79 
81  current_chunk_offset_ += levels_read;
82  }
83  }
RejectedRowIndices invalid_indices_
void processGeoElement(std::string_view geo_string_view)
void appendBaseAndRenderGroupDataAndUpdateMetadata(const int64_t row_count)
#define CHECK(condition)
Definition: Logger.h:289

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ParquetGeospatialEncoder::appendDataTrackErrors ( const int16_t *  def_levels,
const int16_t *  rep_levels,
const int64_t  values_read,
const int64_t  levels_read,
int8_t *  values 
)
inlineoverridevirtual

Implements foreign_storage::ParquetEncoder.

Definition at line 85 of file ParquetGeospatialEncoder.h.

References appendData(), CHECK, and foreign_storage::ParquetEncoder::is_error_tracking_enabled_.

89  {
91  // `appendData` modifies its behaviour based on the
92  // `is_error_tracking_enabled_` flag to handle this case
93  appendData(def_levels, rep_levels, values_read, levels_read, values);
94  }
void appendData(const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override
#define CHECK(condition)
Definition: Logger.h:289

+ Here is the call graph for this function:


The documentation for this class was generated from the following file: