OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Public Member Functions | Private Member Functions | List of all members
foreign_storage::ParquetFixedLengthEncoder< V, T, NullType > Class Template Reference

#include <ParquetFixedLengthEncoder.h>

+ Inheritance diagram for foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >:
+ Collaboration diagram for foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >:

Public Member Functions

 ParquetFixedLengthEncoder (Data_Namespace::AbstractBuffer *buffer, const ColumnDescriptor *column_desciptor, const parquet::ColumnDescriptor *parquet_column_descriptor)
 
 ParquetFixedLengthEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)
 
void encodeAndCopy (const int8_t *parquet_data_bytes, int8_t *omnisci_data_bytes) override
 
void validate (std::shared_ptr< parquet::Statistics > stats, const SQLTypeInfo &column_type) const override
 
void validate (const int8_t *parquet_data, const int64_t j, const SQLTypeInfo &column_type) const override
 
bool encodingIsIdentityForSameTypes () const override
 
- Public Member Functions inherited from foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >
 TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const ColumnDescriptor *column_desciptor, const parquet::ColumnDescriptor *parquet_column_descriptor)
 
 TypedParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)
 
void validate (const int8_t *parquet_data, const int64_t j, const SQLTypeInfo &column_type) const override
 
std::string integralTypeToString (const V &element) const
 
bool isIntegralType (const SQLTypeInfo &type) const
 
std::string elementToString (const V &element) const
 
std::string encodedDataToString (const int8_t *bytes) const override
 
void setDetectBufferConverterType ()
 
void validateUsingEncodersColumnType (const int8_t *parquet_data, const int64_t j) const override
 
void reserve (const size_t num_append_elements) override
 
void appendDataTrackErrors (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override
 
void validateAndAppendData (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices) override
 
void eraseInvalidIndicesInBuffer (const InvalidRowGroupIndices &invalid_indices) override
 
void appendData (const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values) override
 
void encodeAndCopyContiguous (const int8_t *parquet_data_bytes, int8_t *omnisci_data_bytes, const size_t num_elements) override
 
void setNull (int8_t *omnisci_data_bytes) override
 
void copy (const int8_t *omnisci_data_bytes_source, int8_t *omnisci_data_bytes_destination) override
 
std::shared_ptr< ChunkMetadata > getRowGroupMetadata (const parquet::RowGroupMetaData *group_metadata, const int parquet_column_index, const SQLTypeInfo &column_type) override
 
- Public Member Functions inherited from foreign_storage::ParquetInPlaceEncoder
 ParquetInPlaceEncoder (Data_Namespace::AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size)
 
- Public Member Functions inherited from foreign_storage::ParquetScalarEncoder
 ParquetScalarEncoder (Data_Namespace::AbstractBuffer *buffer)
 
- Public Member Functions inherited from foreign_storage::ParquetEncoder
 ParquetEncoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~ParquetEncoder ()=default
 
RejectedRowIndices getRejectedRowIndices () const
 
virtual void disableMetadataStatsValidation ()
 
virtual void initializeErrorTracking ()
 
virtual void initializeColumnType (const SQLTypeInfo &column_type)
 
- Public Member Functions inherited from foreign_storage::ParquetMetadataValidator
virtual ~ParquetMetadataValidator ()=default
 

Private Member Functions

template<typename TT = T, std::enable_if_t<(!std::is_integral< TT >::value||std::is_same< TT, bool >::value)&&!std::is_floating_point< TT >::value, int > = 0>
void validateIntegralOrFloatingPointValue (const T &value, const SQLTypeInfo &column_type) const
 
template<typename TT = T, std::enable_if_t< std::is_floating_point< TT >::value, int > = 0>
void validateIntegralOrFloatingPointValue (const T &value, const SQLTypeInfo &column_type) const
 
template<typename TT = T, std::enable_if_t< std::is_integral< TT >::value &&!std::is_same< TT, bool >::value, int > = 0>
void validateIntegralOrFloatingPointValue (const T &value, const SQLTypeInfo &column_type) const
 
void validateIntegralOrFloatingPointMetadata (std::shared_ptr< parquet::Statistics > stats, const SQLTypeInfo &column_type) const
 

Additional Inherited Members

- Protected Member Functions inherited from foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >
std::pair< T, T > getUnencodedStats (std::shared_ptr< parquet::Statistics > stats) const
 
- Static Protected Member Functions inherited from foreign_storage::ParquetEncoder
static std::shared_ptr
< ChunkMetadata > 		createMetadata (const SQLTypeInfo &column_type)
 
static void throwNotNullViolation (const std::string &parquet_column_name)
 
static void validateNullCount (const std::string &parquet_column_name, int64_t null_count, const SQLTypeInfo &column_type)
 
- Protected Attributes inherited from foreign_storage::ParquetInPlaceEncoder
const size_t omnisci_data_type_byte_size_
 
const size_t parquet_data_type_byte_size_
 
- Protected Attributes inherited from foreign_storage::ParquetEncoder
Data_Namespace::AbstractBuffer * buffer_
 
bool is_error_tracking_enabled_
 
RejectedRowIndices invalid_indices_
 
size_t current_chunk_offset_
 
SQLTypeInfo column_type_
 
bool validate_metadata_stats_
 

Detailed Description

template<typename V, typename T, typename NullType = V>
class foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >

Definition at line 39 of file ParquetFixedLengthEncoder.h.

Constructor & Destructor Documentation

template<typename V , typename T , typename NullType = V>
foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::ParquetFixedLengthEncoder ( Data_Namespace::AbstractBuffer *  buffer,
const ColumnDescriptor *  column_desciptor,
const parquet::ColumnDescriptor *  parquet_column_descriptor 
)
inline

Definition at line 42 of file ParquetFixedLengthEncoder.h.

45  : TypedParquetInPlaceEncoder<V, T, NullType>(buffer,
46  column_desciptor,
47  parquet_column_descriptor) {}
template<typename V , typename T , typename NullType = V>
foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::ParquetFixedLengthEncoder ( Data_Namespace::AbstractBuffer *  buffer,
const size_t  omnisci_data_type_byte_size,
const size_t  parquet_data_type_byte_size 
)
inline

Definition at line 49 of file ParquetFixedLengthEncoder.h.

52  : TypedParquetInPlaceEncoder<V, T, NullType>(buffer,
53  omnisci_data_type_byte_size,
54  parquet_data_type_byte_size) {}

Member Function Documentation

template<typename V , typename T , typename NullType = V>
void foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::encodeAndCopy ( const int8_t *  parquet_data_bytes,
int8_t *  omnisci_data_bytes 
)
inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 56 of file ParquetFixedLengthEncoder.h.

References heavydb.dtypes::T.

57  {
58  const auto& parquet_data_value = reinterpret_cast<const T*>(parquet_data_bytes)[0];
59  auto& omnisci_data_value = reinterpret_cast<V*>(omnisci_data_bytes)[0];
60  omnisci_data_value = parquet_data_value;
61  }
template<typename V , typename T , typename NullType = V>
bool foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::encodingIsIdentityForSameTypes ( ) const
inlineoverridevirtual

Reimplemented from foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >.

Definition at line 75 of file ParquetFixedLengthEncoder.h.

75 { return true; }
template<typename V , typename T , typename NullType = V>
void foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validate ( std::shared_ptr< parquet::Statistics >  stats,
const SQLTypeInfo &  column_type 
) const
inlineoverridevirtual

Implements foreign_storage::ParquetMetadataValidator.

Definition at line 63 of file ParquetFixedLengthEncoder.h.

References foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointMetadata().

64  {
65  validateIntegralOrFloatingPointMetadata(stats, column_type);
66  }
foreign_storage::ParquetFixedLengthEncoder::validateIntegralOrFloatingPointMetadata
void validateIntegralOrFloatingPointMetadata(std::shared_ptr< parquet::Statistics > stats, const SQLTypeInfo &column_type) const
Definition: ParquetFixedLengthEncoder.h:114
report.stats
dictionary stats
Definition: report.py:116

+ Here is the call graph for this function:

template<typename V , typename T , typename NullType = V>
void foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validate ( const int8_t *  parquet_data,
const int64_t  j,
const SQLTypeInfo &  column_type 
) const
inlineoverridevirtual

Implements foreign_storage::ParquetScalarEncoder.

Definition at line 68 of file ParquetFixedLengthEncoder.h.

References heavydb.dtypes::T, and foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointValue().

70  {
71  const auto& parquet_data_value = reinterpret_cast<const T*>(parquet_data)[j];
72  validateIntegralOrFloatingPointValue(parquet_data_value, column_type);
73  }
foreign_storage::ParquetFixedLengthEncoder::validateIntegralOrFloatingPointValue
void validateIntegralOrFloatingPointValue(const T &value, const SQLTypeInfo &column_type) const
Definition: ParquetFixedLengthEncoder.h:83

+ Here is the call graph for this function:

template<typename V , typename T , typename NullType = V>
void foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointMetadata ( std::shared_ptr< parquet::Statistics >  stats,
const SQLTypeInfo &  column_type 
) const
inlineprivate

Definition at line 114 of file ParquetFixedLengthEncoder.h.

References foreign_storage::TypedParquetInPlaceEncoder< V, T, NullType >::getUnencodedStats(), SQLTypeInfo::is_fp(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_timestamp(), and foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointValue().

Referenced by foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validate().

115  {
116  if (!column_type.is_integer() && !column_type.is_timestamp() &&
117  !column_type.is_fp()) {
118  return;
119  }
120  auto [unencoded_stats_min, unencoded_stats_max] =
121  TypedParquetInPlaceEncoder<V, T, NullType>::getUnencodedStats(stats);
122  validateIntegralOrFloatingPointValue(unencoded_stats_min, column_type);
123  validateIntegralOrFloatingPointValue(unencoded_stats_max, column_type);
124  }
foreign_storage::ParquetFixedLengthEncoder::validateIntegralOrFloatingPointValue
void validateIntegralOrFloatingPointValue(const T &value, const SQLTypeInfo &column_type) const
Definition: ParquetFixedLengthEncoder.h:83
SQLTypeInfo::is_timestamp
bool is_timestamp() const
Definition: sqltypes.h:1044
SQLTypeInfo::is_fp
bool is_fp() const
Definition: sqltypes.h:571
report.stats
dictionary stats
Definition: report.py:116
SQLTypeInfo::is_integer
bool is_integer() const
Definition: sqltypes.h:565
foreign_storage::TypedParquetInPlaceEncoder::getUnencodedStats
std::pair< T, T > getUnencodedStats(std::shared_ptr< parquet::Statistics > stats) const
Definition: ParquetInPlaceEncoder.h:364

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename V , typename T , typename NullType = V>
template<typename TT = T, std::enable_if_t<(!std::is_integral< TT >::value||std::is_same< TT, bool >::value)&&!std::is_floating_point< TT >::value, int > = 0>
void foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointValue ( const T &  value,
const SQLTypeInfo &  column_type 
) const
inlineprivate

Definition at line 83 of file ParquetFixedLengthEncoder.h.

Referenced by foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validate(), and foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointMetadata().

84  {
85  // do nothing when type `T` is non-integral and non-floating-point (case
86  // for which this can happen are when `T` is bool)
87  }

+ Here is the caller graph for this function:

template<typename V , typename T , typename NullType = V>
template<typename TT = T, std::enable_if_t< std::is_floating_point< TT >::value, int > = 0>
void foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointValue ( const T &  value,
const SQLTypeInfo &  column_type 
) const
inlineprivate

Definition at line 90 of file ParquetFixedLengthEncoder.h.

References SQLTypeInfo::is_fp(), UNREACHABLE, and foreign_storage::FloatPointValidator< T >::validateValue().

91  {
92  if (column_type.is_fp()) {
93  FloatPointValidator<T>::validateValue(value, column_type);
94  } else {
95  UNREACHABLE();
96  }
97  }
SQLTypeInfo::is_fp
bool is_fp() const
Definition: sqltypes.h:571
UNREACHABLE
#define UNREACHABLE()
Definition: Logger.h:338
foreign_storage::FloatPointValidator::validateValue
static void validateValue(const D &data_value, const SQLTypeInfo &column_type)
Definition: ParquetMetadataValidator.h:286

+ Here is the call graph for this function:

template<typename V , typename T , typename NullType = V>
template<typename TT = T, std::enable_if_t< std::is_integral< TT >::value &&!std::is_same< TT, bool >::value, int > = 0>
void foreign_storage::ParquetFixedLengthEncoder< V, T, NullType >::validateIntegralOrFloatingPointValue ( const T &  value,
const SQLTypeInfo &  column_type 
) const
inlineprivate

Definition at line 103 of file ParquetFixedLengthEncoder.h.

References SQLTypeInfo::is_date(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_timestamp(), foreign_storage::TimestampBoundsValidator< T >::validateValue(), foreign_storage::IntegralFixedLengthBoundsValidator< T >::validateValue(), and foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::validateValue().

104  {
105  if (column_type.is_integer()) {
106  IntegralFixedLengthBoundsValidator<T>::validateValue(value, column_type);
107  } else if (column_type.is_timestamp()) {
108  TimestampBoundsValidator<T>::validateValue(value, column_type);
109  } else if (column_type.is_date()) {
110  DateInDaysBoundsValidator<T>::validateValue(value, column_type);
111  }
112  }
SQLTypeInfo::is_timestamp
bool is_timestamp() const
Definition: sqltypes.h:1044
foreign_storage::BaseDateBoundsValidator::validateValue
static void validateValue(const D &data_value, const SQLTypeInfo &column_type)
Definition: ParquetMetadataValidator.h:216
foreign_storage::IntegralFixedLengthBoundsValidator::validateValue
static void validateValue(const D &data_value, const SQLTypeInfo &column_type)
Definition: ParquetMetadataValidator.h:120
SQLTypeInfo::is_integer
bool is_integer() const
Definition: sqltypes.h:565
SQLTypeInfo::is_date
bool is_date() const
Definition: sqltypes.h:1026
foreign_storage::TimestampBoundsValidator::validateValue
static void validateValue(const D &data_value, const SQLTypeInfo &column_type)
Definition: ParquetMetadataValidator.h:66

+ Here is the call graph for this function:

The documentation for this class was generated from the following file: