foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp} Namespace Reference

Classes
struct	MaxRowGroupSizeStats

Functions
bool	within_range (int64_t lower_bound, int64_t upper_bound, int64_t value)
bool	is_valid_parquet_string (const parquet::ColumnDescriptor *parquet_column)
bool	is_valid_parquet_list_column (const parquet::ColumnDescriptor *parquet_column)
	Detect a valid list parquet column. More...
template<typename V , typename NullType >
std::shared_ptr< ParquetEncoder >	create_parquet_decimal_encoder_with_omnisci_type (const ColumnDescriptor *column_descriptor, const parquet::ColumnDescriptor *parquet_column_descriptor, AbstractBuffer *buffer)
std::shared_ptr< ParquetEncoder >	create_parquet_decimal_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer, const bool is_metadata_scan_or_for_import)
template<typename V , typename T , typename U , typename NullType >
std::shared_ptr< ParquetEncoder >	create_parquet_signed_or_unsigned_integral_encoder_with_types (AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size, const bool is_signed)
	Create a signed or unsigned integral parquet encoder using types. More...
template<typename V , typename NullType >
std::shared_ptr< ParquetEncoder >	create_parquet_integral_encoder_with_omnisci_type (AbstractBuffer *buffer, const size_t omnisci_data_type_byte_size, const size_t parquet_data_type_byte_size, const int bit_width, const bool is_signed)
	Create a integral parquet encoder using types. More...
std::shared_ptr< ParquetEncoder >	create_parquet_integral_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer, const bool is_metadata_scan_or_for_import)
std::shared_ptr< ParquetEncoder >	create_parquet_floating_point_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer)
std::shared_ptr< ParquetEncoder >	create_parquet_none_type_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer)
template<typename V , typename T , typename NullType >
std::shared_ptr< ParquetEncoder >	create_parquet_timestamp_encoder_with_types (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer)
template<typename V , typename T , typename NullType >
std::shared_ptr< ParquetEncoder >	create_parquet_date_from_timestamp_encoder_with_types (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer, const bool is_metadata_scan_or_for_import)
std::shared_ptr< ParquetEncoder >	create_parquet_timestamp_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer, const bool is_metadata_scan_or_for_import)
template<typename V , typename T , typename NullType >
std::shared_ptr< ParquetEncoder >	create_parquet_time_encoder_with_types (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer)
std::shared_ptr< ParquetEncoder >	create_parquet_time_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer, const bool is_metadata_scan_or_for_import)
std::shared_ptr< ParquetEncoder >	create_parquet_date_from_timestamp_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer, const bool is_metadata_scan_or_for_import)
std::shared_ptr< ParquetEncoder >	create_parquet_date_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, AbstractBuffer *buffer, const bool is_metadata_scan_or_for_import)
std::shared_ptr< ParquetEncoder >	create_parquet_string_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, const Chunk_NS::Chunk &chunk, StringDictionary *string_dictionary, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, bool is_for_import, const bool is_for_detect)
std::shared_ptr< ParquetEncoder >	create_parquet_geospatial_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, std::list< Chunk_NS::Chunk > &chunks, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const bool is_metadata_scan, const bool is_for_import, const bool geo_validate_geometry)
std::shared_ptr< ParquetEncoder >	create_parquet_array_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, std::list< Chunk_NS::Chunk > &chunks, StringDictionary *string_dictionary, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const bool is_metadata_scan, const bool is_for_import, const bool is_for_detect, const bool geo_validate_geometry)
std::shared_ptr< ParquetEncoder >	create_parquet_encoder (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, std::list< Chunk_NS::Chunk > &chunks, StringDictionary *string_dictionary, std::list< std::unique_ptr< ChunkMetadata >> &chunk_metadata, const bool is_metadata_scan, const bool is_for_import, const bool is_for_detect, const bool geo_validate_geometry)
	Create a Parquet specific encoder for a Parquet to OmniSci mapping. More...
std::shared_ptr< ParquetEncoder >	create_parquet_encoder_for_import (std::list< Chunk_NS::Chunk > &chunks, const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, StringDictionary *string_dictionary, const bool geo_validate_geometry)
std::shared_ptr< ParquetEncoder >	create_parquet_encoder_for_metadata_scan (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, const bool geo_validate_geometry)
void	validate_list_column_metadata_statistics (const parquet::ParquetFileReader *reader, const int row_group_index, const int column_index, const int16_t *def_levels, const int64_t num_levels, const parquet::ColumnDescriptor *parquet_column_descriptor)
void	set_definition_levels_for_zero_max_definition_level_case (const parquet::ColumnDescriptor *parquet_column_descriptor, std::vector< int16_t > &def_levels)
void	validate_max_repetition_and_definition_level (const ColumnDescriptor *omnisci_column_descriptor, const parquet::ColumnDescriptor *parquet_column_descriptor)
void	resize_values_buffer (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, std::vector< int8_t > &values)
bool	validate_decimal_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_decimal_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	validate_floating_point_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_floating_point_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	validate_integral_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_integral_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	is_nanosecond_precision (const ColumnDescriptor *omnisci_column)
bool	is_nanosecond_precision (const parquet::TimestampLogicalType *timestamp_logical_column)
bool	is_microsecond_precision (const ColumnDescriptor *omnisci_column)
bool	is_microsecond_precision (const parquet::TimestampLogicalType *timestamp_logical_column)
bool	is_millisecond_precision (const ColumnDescriptor *omnisci_column)
bool	is_millisecond_precision (const parquet::TimestampLogicalType *timestamp_logical_column)
bool	validate_none_type_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_boolean_type_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	validate_timestamp_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_timestamp_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	validate_time_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_time_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	validate_date_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_date_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	validate_string_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
SQLTypeInfo	suggest_string_mapping (const parquet::ColumnDescriptor *parquet_column)
bool	validate_geospatial_mapping (const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column)
void	validate_equal_schema (const parquet::arrow::FileReader *reference_file_reader, const parquet::arrow::FileReader *new_file_reader, const std::string &reference_file_path, const std::string &new_file_path)
void	validate_allowed_mapping (const parquet::ColumnDescriptor *parquet_column, const ColumnDescriptor *omnisci_column)
SQLTypeInfo	suggest_column_scalar_type (const parquet::ColumnDescriptor *parquet_column)
void	validate_number_of_columns (const std::shared_ptr< parquet::FileMetaData > &file_metadata, const std::string &file_path, const ForeignTableSchema &schema)
void	throw_missing_metadata_error (const int row_group_index, const int column_index, const std::string &file_path)
void	throw_row_group_larger_than_fragment_size_error (const MaxRowGroupSizeStats max_row_group_stats, const int fragment_size)
MaxRowGroupSizeStats	validate_column_mapping_and_row_group_metadata (const std::shared_ptr< parquet::FileMetaData > &file_metadata, const std::string &file_path, const ForeignTableSchema &schema, const bool do_metadata_stats_validation)
MaxRowGroupSizeStats	validate_parquet_metadata (const std::shared_ptr< parquet::FileMetaData > &file_metadata, const std::string &file_path, const ForeignTableSchema &schema, const bool do_metadata_stats_validation)
std::list< RowGroupMetadata >	metadata_scan_rowgroup_interval (const std::map< int, std::shared_ptr< ParquetEncoder >> &encoder_map, const RowGroupInterval &row_group_interval, const ReaderPtr &reader, const ForeignTableSchema &schema)
std::map< int, std::shared_ptr < ParquetEncoder > >	populate_encoder_map_for_import (const std::map< int, Chunk_NS::Chunk > chunks, const ForeignTableSchema &schema, const ReaderPtr &reader, const std::map< int, StringDictionary * > column_dictionaries, const int64_t num_rows, const bool geo_validate_geometry)
std::map< int, std::shared_ptr < ParquetEncoder > >	populate_encoder_map_for_metadata_scan (const Interval< ColumnType > &column_interval, const ForeignTableSchema &schema, const ReaderPtr &reader, const bool do_metadata_stats_validation, const bool geo_validate_geometry)

Function Documentation

std::shared_ptr< ParquetEncoder > foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_array_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		std::list< Chunk_NS::Chunk > &	chunks,
		StringDictionary *	string_dictionary,
		std::list< std::unique_ptr< ChunkMetadata >> &	chunk_metadata,
		const bool	is_metadata_scan,
		const bool	is_for_import,
		const bool	is_for_detect,
		const bool	geo_validate_geometry
	)

Definition at line 1025 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnType, create_parquet_encoder(), foreign_storage::get_sub_type_column_descriptor(), SQLTypeInfo::is_array(), SQLTypeInfo::is_fixlen_array(), and is_valid_parquet_list_column().

Referenced by create_parquet_encoder().

                                      {
  bool is_valid_parquet_list = is_valid_parquet_list_column(parquet_column);
  if (!is_valid_parquet_list || !omnisci_column->columnType.is_array()) {
    return {};
  }
  std::unique_ptr<ColumnDescriptor> omnisci_column_sub_type_column =
      get_sub_type_column_descriptor(omnisci_column);
  auto encoder = create_parquet_encoder(omnisci_column_sub_type_column.get(),
                                        parquet_column,
                                        chunks,
                                        string_dictionary,
                                        chunk_metadata,
                                        is_metadata_scan,
                                        is_for_import,
                                        is_for_detect,
                                        geo_validate_geometry);
  CHECK(encoder.get());
  auto scalar_encoder = std::dynamic_pointer_cast<ParquetScalarEncoder>(encoder);
  CHECK(scalar_encoder);
  if (!is_for_import) {
    if (!is_for_detect) {
      if (omnisci_column->columnType.is_fixlen_array()) {
        encoder = std::make_shared<ParquetFixedLengthArrayEncoder>(
            is_metadata_scan ? nullptr : chunks.begin()->getBuffer(),
            scalar_encoder,
            omnisci_column);
      } else {
        encoder = std::make_shared<ParquetVariableLengthArrayEncoder>(
            is_metadata_scan ? nullptr : chunks.begin()->getBuffer(),
            is_metadata_scan ? nullptr : chunks.begin()->getIndexBuf(),
            scalar_encoder,
            omnisci_column);
      }
    } else {  // is_for_detect
      encoder = std::make_shared<ParquetArrayDetectEncoder>(
          chunks.begin()->getBuffer(), scalar_encoder, omnisci_column);
    }
  } else {  // is_for_import
    encoder = std::make_shared<ParquetArrayImportEncoder>(
        chunks.begin()->getBuffer(), scalar_encoder, omnisci_column);
  }
  return encoder;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_date_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer,
		const bool	is_metadata_scan_or_for_import
	)

Definition at line 737 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, kENCODING_DATE_IN_DAYS, kENCODING_NONE, and UNREACHABLE.

Referenced by create_parquet_encoder().

                                               {
  auto column_type = omnisci_column->columnType;
  if (parquet_column->logical_type()->is_date() && column_type.is_date()) {
    if (column_type.get_compression() == kENCODING_DATE_IN_DAYS) {
      if (is_metadata_scan_or_for_import) {
        if (column_type.get_comp_param() ==
            0) {  // DATE ENCODING FIXED (32) uses comp param 0
          return std::make_shared<ParquetDateInSecondsEncoder</*NullType=*/int32_t>>(
              buffer);
        } else if (column_type.get_comp_param() == 16) {
          return std::make_shared<ParquetDateInSecondsEncoder</*NullType=*/int16_t>>(
              buffer);
        } else {
          UNREACHABLE();
        }
      } else {
        if (column_type.get_comp_param() ==
            0) {  // DATE ENCODING FIXED (32) uses comp param 0
          return std::make_shared<ParquetFixedLengthEncoder<int32_t, int32_t>>(
              buffer, omnisci_column, parquet_column);
        } else if (column_type.get_comp_param() == 16) {
          return std::make_shared<ParquetFixedLengthEncoder<int16_t, int32_t>>(
              buffer, omnisci_column, parquet_column);
        } else {
          UNREACHABLE();
        }
      }
    } else if (column_type.get_compression() == kENCODING_NONE) {  // for array types
      return std::make_shared<ParquetDateInSecondsEncoder</*NullType=*/int64_t>>(
          buffer, omnisci_column, parquet_column);
    } else {
      UNREACHABLE();
    }
  }
  return {};
}

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_date_from_timestamp_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer,
		const bool	is_metadata_scan_or_for_import
	)

Definition at line 694 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnType, create_parquet_date_from_timestamp_encoder_with_types(), kENCODING_DATE_IN_DAYS, and UNREACHABLE.

Referenced by create_parquet_encoder().

                                               {
  auto column_type = omnisci_column->columnType;
  if (parquet_column->logical_type()->is_timestamp() && column_type.is_date()) {
    CHECK(column_type.get_compression() == kENCODING_DATE_IN_DAYS);
    if (is_metadata_scan_or_for_import) {
      if (column_type.get_comp_param() ==
          0) {  // DATE ENCODING FIXED (32) uses comp param 0
        return create_parquet_date_from_timestamp_encoder_with_types<int64_t,
                                                                     int64_t,
                                                                     int32_t>(
            omnisci_column, parquet_column, buffer, true);
      } else if (column_type.get_comp_param() == 16) {
        return create_parquet_date_from_timestamp_encoder_with_types<int64_t,
                                                                     int64_t,
                                                                     int16_t>(
            omnisci_column, parquet_column, buffer, true);
      } else {
        UNREACHABLE();
      }
    } else {
      if (column_type.get_comp_param() ==
          0) {  // DATE ENCODING FIXED (32) uses comp param 0
        return create_parquet_date_from_timestamp_encoder_with_types<int32_t,
                                                                     int64_t,
                                                                     int32_t>(
            omnisci_column, parquet_column, buffer, false);
      } else if (column_type.get_comp_param() == 16) {
        return create_parquet_date_from_timestamp_encoder_with_types<int16_t,
                                                                     int64_t,
                                                                     int16_t>(
            omnisci_column, parquet_column, buffer, false);
      } else {
        UNREACHABLE();
      }
    }
  }
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V , typename T , typename NullType >

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_date_from_timestamp_encoder_with_types	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer,
		const bool	is_metadata_scan_or_for_import
	)

Definition at line 509 of file LazyParquetChunkLoader.cpp.

References heavydb.dtypes::T, and UNREACHABLE.

Referenced by create_parquet_date_from_timestamp_encoder().

                                               {
  if (auto timestamp_logical_type = dynamic_cast<const parquet::TimestampLogicalType*>(
          parquet_column->logical_type().get())) {
    switch (timestamp_logical_type->time_unit()) {
      case parquet::LogicalType::TimeUnit::MILLIS:
        if (is_metadata_scan_or_for_import) {
          return std::make_shared<
              ParquetDateInSecondsFromTimestampEncoder<V, T, 1000L, NullType>>(
              buffer, omnisci_column, parquet_column);
        }
        return std::make_shared<
            ParquetDateInDaysFromTimestampEncoder<V, T, 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      case parquet::LogicalType::TimeUnit::MICROS:
        if (is_metadata_scan_or_for_import) {
          return std::make_shared<
              ParquetDateInSecondsFromTimestampEncoder<V, T, 1000L * 1000L, NullType>>(
              buffer, omnisci_column, parquet_column);
        }
        return std::make_shared<
            ParquetDateInDaysFromTimestampEncoder<V, T, 1000L * 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      case parquet::LogicalType::TimeUnit::NANOS:
        if (is_metadata_scan_or_for_import) {
          return std::make_shared<
              ParquetDateInSecondsFromTimestampEncoder<V,
                                                       T,
                                                       1000L * 1000L * 1000L,
                                                       NullType>>(
              buffer, omnisci_column, parquet_column);
        }
        return std::make_shared<
            ParquetDateInDaysFromTimestampEncoder<V, T, 1000L * 1000L * 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      default:
        UNREACHABLE();
    }
  } else {
    UNREACHABLE();
  }
  return {};
}

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_decimal_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer,
		const bool	is_metadata_scan_or_for_import
	)

Definition at line 172 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), kENCODING_FIXED, kENCODING_NONE, and UNREACHABLE.

Referenced by create_parquet_encoder().

                                               {
  if (parquet_column->logical_type()->is_decimal()) {
    if (omnisci_column->columnType.get_compression() == kENCODING_NONE) {
      return create_parquet_decimal_encoder_with_omnisci_type<int64_t, int64_t>(
          omnisci_column, parquet_column, buffer);
    }
    CHECK(omnisci_column->columnType.get_compression() == kENCODING_FIXED);
    if (is_metadata_scan_or_for_import) {
      switch (omnisci_column->columnType.get_comp_param()) {
        case 16:
          return create_parquet_decimal_encoder_with_omnisci_type<int64_t, int16_t>(
              omnisci_column, parquet_column, buffer);
        case 32:
          return create_parquet_decimal_encoder_with_omnisci_type<int64_t, int32_t>(
              omnisci_column, parquet_column, buffer);
        default:
          UNREACHABLE();
      }
    } else {
      switch (omnisci_column->columnType.get_comp_param()) {
        case 16:
          return create_parquet_decimal_encoder_with_omnisci_type<int16_t, int16_t>(
              omnisci_column, parquet_column, buffer);
        case 32:
          return create_parquet_decimal_encoder_with_omnisci_type<int32_t, int32_t>(
              omnisci_column, parquet_column, buffer);
        default:
          UNREACHABLE();
      }
    }
  }
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V , typename NullType >

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_decimal_encoder_with_omnisci_type	(	const ColumnDescriptor *	column_descriptor,
		const parquet::ColumnDescriptor *	parquet_column_descriptor,
		AbstractBuffer *	buffer
	)

Definition at line 148 of file LazyParquetChunkLoader.cpp.

References UNREACHABLE.

                            {
  switch (parquet_column_descriptor->physical_type()) {
    case parquet::Type::INT32:
      return std::make_shared<ParquetDecimalEncoder<V, int32_t, NullType>>(
          buffer, column_descriptor, parquet_column_descriptor);
    case parquet::Type::INT64:
      return std::make_shared<ParquetDecimalEncoder<V, int64_t, NullType>>(
          buffer, column_descriptor, parquet_column_descriptor);
    case parquet::Type::FIXED_LEN_BYTE_ARRAY:
      return std::make_shared<
          ParquetDecimalEncoder<V, parquet::FixedLenByteArray, NullType>>(
          buffer, column_descriptor, parquet_column_descriptor);
    case parquet::Type::BYTE_ARRAY:
      return std::make_shared<ParquetDecimalEncoder<V, parquet::ByteArray, NullType>>(
          buffer, column_descriptor, parquet_column_descriptor);
    default:
      UNREACHABLE();
  }
  return {};
}

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		std::list< Chunk_NS::Chunk > &	chunks,
		StringDictionary *	string_dictionary,
		std::list< std::unique_ptr< ChunkMetadata >> &	chunk_metadata,
		const bool	is_metadata_scan,
		const bool	is_for_import,
		const bool	is_for_detect,
		const bool	geo_validate_geometry
	)

Create a Parquet specific encoder for a Parquet to OmniSci mapping.

Parameters

omnisci_column	- the descriptor of OmniSci column
parquet_column	- the descriptor of Parquet column
chunks	- list of chunks to populate (the case of more than one chunk happens only if a logical column expands to multiple physical columns)
string_dictionary	- string dictionary used in encoding for string dictionary encoded columns
chunk_metadata	- similar to the list of chunks, a list of chunk metadata that is populated
is_metadata_scan	- a flag indicating if the encoders created should be for a metadata scan
is_for_import	- a flag indicating if the encoders created should be for import

Returns

An appropriate Parquet encoder for the use case defined by the Parquet to OmniSci mapping.

Notes:

• In the case of a metadata scan, the type of the encoder created may significantly change (for example in bit width.) This is because it is common for OmniSci to store metadata in a different format altogether than the data itself (see for example FixedLengthEncoder.)
• This function and the function isColumnMappingSupported work in conjunction with each other. For example, once a mapping is known to be allowed (since isColumnMappingSupported returned true) this function does not have to check many corner cases exhaustively as it would be redundant with what was checked in isColumnMappingSupported.

Definition at line 905 of file LazyParquetChunkLoader.cpp.

References CHECK, create_parquet_array_encoder(), create_parquet_date_encoder(), create_parquet_date_from_timestamp_encoder(), create_parquet_decimal_encoder(), create_parquet_floating_point_encoder(), create_parquet_geospatial_encoder(), create_parquet_integral_encoder(), create_parquet_none_type_encoder(), create_parquet_string_encoder(), create_parquet_time_encoder(), create_parquet_timestamp_encoder(), and UNREACHABLE.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), create_parquet_array_encoder(), create_parquet_encoder_for_import(), and create_parquet_encoder_for_metadata_scan().

                                      {
  CHECK(!(is_metadata_scan && is_for_import));
  auto buffer = chunks.empty() ? nullptr : chunks.begin()->getBuffer();
  if (auto encoder = create_parquet_geospatial_encoder(omnisci_column,
                                                       parquet_column,
                                                       chunks,
                                                       chunk_metadata,
                                                       is_metadata_scan,
                                                       is_for_import,
                                                       geo_validate_geometry)) {
    return encoder;
  }
  if (auto encoder = create_parquet_array_encoder(omnisci_column,
                                                  parquet_column,
                                                  chunks,
                                                  string_dictionary,
                                                  chunk_metadata,
                                                  is_metadata_scan,
                                                  is_for_import,
                                                  is_for_detect,
                                                  geo_validate_geometry)) {
    return encoder;
  }
  if (auto encoder = create_parquet_decimal_encoder(
          omnisci_column, parquet_column, buffer, is_metadata_scan || is_for_import)) {
    return encoder;
  }
  if (auto encoder = create_parquet_integral_encoder(
          omnisci_column, parquet_column, buffer, is_metadata_scan || is_for_import)) {
    return encoder;
  }
  if (auto encoder =
          create_parquet_floating_point_encoder(omnisci_column, parquet_column, buffer)) {
    return encoder;
  }
  if (auto encoder = create_parquet_timestamp_encoder(
          omnisci_column, parquet_column, buffer, is_metadata_scan || is_for_import)) {
    return encoder;
  }
  if (auto encoder =
          create_parquet_none_type_encoder(omnisci_column, parquet_column, buffer)) {
    return encoder;
  }
  if (auto encoder = create_parquet_time_encoder(
          omnisci_column, parquet_column, buffer, is_metadata_scan || is_for_import)) {
    return encoder;
  }
  if (auto encoder = create_parquet_date_from_timestamp_encoder(
          omnisci_column, parquet_column, buffer, is_metadata_scan || is_for_import)) {
    return encoder;
  }
  if (auto encoder = create_parquet_date_encoder(
          omnisci_column, parquet_column, buffer, is_metadata_scan || is_for_import)) {
    return encoder;
  }
  if (auto encoder = create_parquet_string_encoder(
          omnisci_column,
          parquet_column,
          chunks.empty() ? Chunk_NS::Chunk{} : *chunks.begin(),
          string_dictionary,
          chunk_metadata,
          is_for_import,
          is_for_detect)) {
    return encoder;
  }
  UNREACHABLE();
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_encoder_for_import	(	std::list< Chunk_NS::Chunk > &	chunks,
		const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		StringDictionary *	string_dictionary,
		const bool	geo_validate_geometry
	)

Intended to be used for the import case.

Definition at line 986 of file LazyParquetChunkLoader.cpp.

References create_parquet_encoder().

Referenced by populate_encoder_map_for_import().

                                      {
  std::list<std::unique_ptr<ChunkMetadata>> chunk_metadata;
  return create_parquet_encoder(omnisci_column,
                                parquet_column,
                                chunks,
                                string_dictionary,
                                chunk_metadata,
                                false,
                                true,
                                false,
                                geo_validate_geometry);
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_encoder_for_metadata_scan	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		const bool	geo_validate_geometry
	)

Intended to be used only with metadata scan. Creates an incomplete encoder capable of updating metadata.

Definition at line 1008 of file LazyParquetChunkLoader.cpp.

References create_parquet_encoder().

Referenced by populate_encoder_map_for_metadata_scan().

                                      {
  std::list<Chunk_NS::Chunk> chunks;
  std::list<std::unique_ptr<ChunkMetadata>> chunk_metadata;
  return create_parquet_encoder(omnisci_column,
                                parquet_column,
                                chunks,
                                nullptr,
                                chunk_metadata,
                                true,
                                false,
                                false,
                                geo_validate_geometry);
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_floating_point_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer
	)

Definition at line 428 of file LazyParquetChunkLoader.cpp.

References CHECK, CHECK_EQ, ColumnDescriptor::columnType, kDOUBLE, kENCODING_NONE, kFLOAT, and UNREACHABLE.

Referenced by create_parquet_encoder().

                            {
  auto column_type = omnisci_column->columnType;
  if (!column_type.is_fp()) {
    return {};
  }
  CHECK_EQ(column_type.get_compression(), kENCODING_NONE);
  switch (column_type.get_type()) {
    case kFLOAT:
      switch (parquet_column->physical_type()) {
        case parquet::Type::FLOAT:
          return std::make_shared<ParquetFixedLengthEncoder<float, float>>(
              buffer, omnisci_column, parquet_column);
        case parquet::Type::DOUBLE:
          return std::make_shared<ParquetFixedLengthEncoder<float, double>>(
              buffer, omnisci_column, parquet_column);
        default:
          UNREACHABLE();
      }
    case kDOUBLE:
      CHECK(parquet_column->physical_type() == parquet::Type::DOUBLE);
      return std::make_shared<ParquetFixedLengthEncoder<double, double>>(
          buffer, omnisci_column, parquet_column);
    default:
      UNREACHABLE();
  }
  return {};
}

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_geospatial_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		std::list< Chunk_NS::Chunk > &	chunks,
		std::list< std::unique_ptr< ChunkMetadata >> &	chunk_metadata,
		const bool	is_metadata_scan,
		const bool	is_for_import,
		const bool	geo_validate_geometry
	)

Definition at line 831 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, and is_valid_parquet_string().

Referenced by create_parquet_encoder().

                                      {
  auto column_type = omnisci_column->columnType;
  if (!is_valid_parquet_string(parquet_column) || !column_type.is_geometry()) {
    return {};
  }
  if (is_for_import) {
    return std::make_shared<ParquetGeospatialImportEncoder>(chunks,
                                                            geo_validate_geometry);
  }
  if (is_metadata_scan) {
    return std::make_shared<ParquetGeospatialEncoder>(geo_validate_geometry);
  }
  for (auto chunks_iter = chunks.begin(); chunks_iter != chunks.end(); ++chunks_iter) {
    chunk_metadata.emplace_back(std::make_unique<ChunkMetadata>());
    auto& chunk_metadata_ptr = chunk_metadata.back();
    chunk_metadata_ptr->sqlType = chunks_iter->getColumnDesc()->columnType;
  }
  return std::make_shared<ParquetGeospatialEncoder>(
      parquet_column, chunks, chunk_metadata, geo_validate_geometry);
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_integral_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer,
		const bool	is_metadata_scan_or_for_import
	)

Definition at line 298 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnType, kBIGINT, kENCODING_NONE, kINT, kSMALLINT, kTINYINT, and UNREACHABLE.

Referenced by create_parquet_encoder().

                                               {
  auto column_type = omnisci_column->columnType;
  auto physical_type = parquet_column->physical_type();

  int bit_width = -1;
  int is_signed = false;
  // handle the integral case with no Parquet annotation
  if (parquet_column->logical_type()->is_none() && column_type.is_integer()) {
    if (physical_type == parquet::Type::INT32) {
      bit_width = 32;
    } else if (physical_type == parquet::Type::INT64) {
      bit_width = 64;
    } else {
      UNREACHABLE();
    }
    is_signed = true;
  }
  // handle the integral case with Parquet annotation
  if (auto int_logical_column = dynamic_cast<const parquet::IntLogicalType*>(
          parquet_column->logical_type().get())) {
    bit_width = int_logical_column->bit_width();
    is_signed = int_logical_column->is_signed();
  }

  if (bit_width == -1) {  // no valid logical type (with or without annotation) found
    return {};
  }

  const size_t omnisci_data_type_byte_size = column_type.get_size();
  const size_t parquet_data_type_byte_size = parquet::GetTypeByteSize(physical_type);

  switch (omnisci_data_type_byte_size) {
    case 8:
      CHECK(column_type.get_compression() == kENCODING_NONE);
      return create_parquet_integral_encoder_with_omnisci_type<int64_t, int64_t>(
          buffer,
          omnisci_data_type_byte_size,
          parquet_data_type_byte_size,
          bit_width,
          is_signed);
    case 4:
      if (is_metadata_scan_or_for_import && column_type.get_type() == kBIGINT) {
        return create_parquet_integral_encoder_with_omnisci_type<int64_t, int32_t>(
            buffer,
            omnisci_data_type_byte_size,
            parquet_data_type_byte_size,
            bit_width,
            is_signed);
      }
      return create_parquet_integral_encoder_with_omnisci_type<int32_t, int32_t>(
          buffer,
          omnisci_data_type_byte_size,
          parquet_data_type_byte_size,
          bit_width,
          is_signed);
    case 2:
      if (is_metadata_scan_or_for_import) {
        switch (column_type.get_type()) {
          case kBIGINT:
            return create_parquet_integral_encoder_with_omnisci_type<int64_t, int16_t>(
                buffer,
                omnisci_data_type_byte_size,
                parquet_data_type_byte_size,
                bit_width,
                is_signed);
          case kINT:
            return create_parquet_integral_encoder_with_omnisci_type<int32_t, int16_t>(
                buffer,
                omnisci_data_type_byte_size,
                parquet_data_type_byte_size,
                bit_width,
                is_signed);
          case kSMALLINT:
            break;
          default:
            UNREACHABLE();
        }
      }
      return create_parquet_integral_encoder_with_omnisci_type<int16_t, int16_t>(
          buffer,
          omnisci_data_type_byte_size,
          parquet_data_type_byte_size,
          bit_width,
          is_signed);
    case 1:
      if (is_metadata_scan_or_for_import) {
        switch (column_type.get_type()) {
          case kBIGINT:
            return create_parquet_integral_encoder_with_omnisci_type<int64_t, int8_t>(
                buffer,
                omnisci_data_type_byte_size,
                parquet_data_type_byte_size,
                bit_width,
                is_signed);
          case kINT:
            return create_parquet_integral_encoder_with_omnisci_type<int32_t, int8_t>(
                buffer,
                omnisci_data_type_byte_size,
                parquet_data_type_byte_size,
                bit_width,
                is_signed);
          case kSMALLINT:
            return create_parquet_integral_encoder_with_omnisci_type<int16_t, int8_t>(
                buffer,
                omnisci_data_type_byte_size,
                parquet_data_type_byte_size,
                bit_width,
                is_signed);
          case kTINYINT:
            break;
          default:
            UNREACHABLE();
        }
      }
      return create_parquet_integral_encoder_with_omnisci_type<int8_t, int8_t>(
          buffer,
          omnisci_data_type_byte_size,
          parquet_data_type_byte_size,
          bit_width,
          is_signed);
    default:
      UNREACHABLE();
  }
  return {};
}

Here is the caller graph for this function:

template<typename V , typename NullType >

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_integral_encoder_with_omnisci_type	(	AbstractBuffer *	buffer,
		const size_t	omnisci_data_type_byte_size,
		const size_t	parquet_data_type_byte_size,
		const int	bit_width,
		const bool	is_signed
	)

Create a integral parquet encoder using types.

Parameters

buffer	- buffer used within the encoder
omnisci_data_type_byte_size	- size in number of bytes of OmniSci type
parquet_data_type_byte_size	- size in number of bytes of Parquet physical type
bit_width	- bit width specified for the Parquet column
is_signed	- flag indicating if Parquet column is signed

Returns

a std::shared_ptr to an integral encoder

See the documentation for ParquetFixedLengthEncoder and ParquetUnsignedFixedLengthEncoder for a description of the semantics of the templated type V and NullType.

Note, this function determines the appropriate bit depth integral encoder to create, while create_parquet_signed_or_unsigned_integral_encoder_with_types determines whether to create a signed or unsigned integral encoder.

Definition at line 261 of file LazyParquetChunkLoader.cpp.

References create_parquet_signed_or_unsigned_integral_encoder_with_types(), and UNREACHABLE.

                          {
  switch (bit_width) {
    case 8:
      return create_parquet_signed_or_unsigned_integral_encoder_with_types<V,
                                                                           int32_t,
                                                                           uint8_t,
                                                                           NullType>(
          buffer, omnisci_data_type_byte_size, parquet_data_type_byte_size, is_signed);
    case 16:
      return create_parquet_signed_or_unsigned_integral_encoder_with_types<V,
                                                                           int32_t,
                                                                           uint16_t,
                                                                           NullType>(
          buffer, omnisci_data_type_byte_size, parquet_data_type_byte_size, is_signed);
    case 32:
      return create_parquet_signed_or_unsigned_integral_encoder_with_types<V,
                                                                           int32_t,
                                                                           uint32_t,
                                                                           NullType>(
          buffer, omnisci_data_type_byte_size, parquet_data_type_byte_size, is_signed);
    case 64:
      return create_parquet_signed_or_unsigned_integral_encoder_with_types<V,
                                                                           int64_t,
                                                                           uint64_t,
                                                                           NullType>(
          buffer, omnisci_data_type_byte_size, parquet_data_type_byte_size, is_signed);
    default:
      UNREACHABLE();
  }
  return {};
}

Here is the call graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_none_type_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer
	)

Definition at line 459 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::is_string(), kBOOLEAN, kENCODING_NONE, and UNREACHABLE.

Referenced by create_parquet_encoder().

                            {
  auto column_type = omnisci_column->columnType;
  if (parquet_column->logical_type()->is_none() &&
      !omnisci_column->columnType.is_string()) {  // boolean
    if (column_type.get_compression() == kENCODING_NONE) {
      switch (column_type.get_type()) {
        case kBOOLEAN:
          return std::make_shared<ParquetFixedLengthEncoder<int8_t, bool>>(
              buffer, omnisci_column, parquet_column);
        default:
          UNREACHABLE();
      }
    } else {
      UNREACHABLE();
    }
  }
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V , typename T , typename U , typename NullType >

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_signed_or_unsigned_integral_encoder_with_types	(	AbstractBuffer *	buffer,
		const size_t	omnisci_data_type_byte_size,
		const size_t	parquet_data_type_byte_size,
		const bool	is_signed
	)

Create a signed or unsigned integral parquet encoder using types.

Parameters

buffer	- buffer used within the encoder
omnisci_data_type_byte_size	- size in number of bytes of OmniSci type
parquet_data_type_byte_size	- size in number of bytes of Parquet physical type
is_signed	- flag indicating if Parquet column is signed

Returns

a std::shared_ptr to an integral encoder

See the documentation for ParquetFixedLengthEncoder and ParquetUnsignedFixedLengthEncoder for a description of the semantics of the templated types V, T, U, and NullType.

Definition at line 226 of file LazyParquetChunkLoader.cpp.

References CHECK.

Referenced by create_parquet_integral_encoder_with_omnisci_type().

                          {
  CHECK(sizeof(NullType) == omnisci_data_type_byte_size);
  if (is_signed) {
    return std::make_shared<ParquetFixedLengthEncoder<V, T, NullType>>(
        buffer, omnisci_data_type_byte_size, parquet_data_type_byte_size);
  } else {
    return std::make_shared<ParquetUnsignedFixedLengthEncoder<V, T, U, NullType>>(
        buffer, omnisci_data_type_byte_size, parquet_data_type_byte_size);
  }
}

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_string_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		const Chunk_NS::Chunk &	chunk,
		StringDictionary *	string_dictionary,
		std::list< std::unique_ptr< ChunkMetadata >> &	chunk_metadata,
		bool	is_for_import,
		const bool	is_for_detect
	)

Definition at line 778 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, Chunk_NS::Chunk::getBuffer(), Chunk_NS::Chunk::getIndexBuf(), SQLTypeInfo::is_string(), is_valid_parquet_string(), kENCODING_DICT, kENCODING_NONE, and UNREACHABLE.

Referenced by create_parquet_encoder().

                              {
  auto column_type = omnisci_column->columnType;
  if (!is_valid_parquet_string(parquet_column) ||
      !omnisci_column->columnType.is_string()) {
    return {};
  }
  if (column_type.get_compression() == kENCODING_NONE) {
    if (is_for_import) {
      return std::make_shared<ParquetStringImportEncoder>(chunk.getBuffer());
    } else {
      return std::make_shared<ParquetStringNoneEncoder>(chunk.getBuffer(),
                                                        chunk.getIndexBuf());
    }
  } else if (column_type.get_compression() == kENCODING_DICT) {
    if (!is_for_detect) {  // non-detect use case
      chunk_metadata.emplace_back(std::make_unique<ChunkMetadata>());
      std::unique_ptr<ChunkMetadata>& logical_chunk_metadata = chunk_metadata.back();
      logical_chunk_metadata->sqlType = omnisci_column->columnType;
      switch (column_type.get_size()) {
        case 1:
          return std::make_shared<ParquetStringEncoder<uint8_t>>(
              chunk.getBuffer(),
              string_dictionary,
              is_for_import ? nullptr : logical_chunk_metadata.get());
        case 2:
          return std::make_shared<ParquetStringEncoder<uint16_t>>(
              chunk.getBuffer(),
              string_dictionary,
              is_for_import ? nullptr : logical_chunk_metadata.get());
        case 4:
          return std::make_shared<ParquetStringEncoder<int32_t>>(
              chunk.getBuffer(),
              string_dictionary,
              is_for_import ? nullptr : logical_chunk_metadata.get());
        default:
          UNREACHABLE();
      }
    } else {  // detect use-case
      return std::make_shared<ParquetDetectStringEncoder>(chunk.getBuffer());
    }
  } else {
    UNREACHABLE();
  }
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_time_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer,
		const bool	is_metadata_scan_or_for_import
	)

Definition at line 641 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnType, kENCODING_FIXED, kENCODING_NONE, and UNREACHABLE.

Referenced by create_parquet_encoder().

                                               {
  auto column_type = omnisci_column->columnType;
  if (auto time_logical_column = dynamic_cast<const parquet::TimeLogicalType*>(
          parquet_column->logical_type().get())) {
    if (column_type.get_compression() == kENCODING_NONE) {
      if (time_logical_column->time_unit() == parquet::LogicalType::TimeUnit::MILLIS) {
        return create_parquet_time_encoder_with_types<int64_t, int32_t, int64_t>(
            omnisci_column, parquet_column, buffer);
      } else {
        return create_parquet_time_encoder_with_types<int64_t, int64_t, int64_t>(
            omnisci_column, parquet_column, buffer);
      }
    } else if (column_type.get_compression() == kENCODING_FIXED) {
      if (is_metadata_scan_or_for_import) {
        if (time_logical_column->time_unit() == parquet::LogicalType::TimeUnit::MILLIS) {
          CHECK(parquet_column->physical_type() == parquet::Type::INT32);
          return create_parquet_time_encoder_with_types<int64_t, int32_t, int32_t>(
              omnisci_column, parquet_column, buffer);
        } else {
          CHECK(time_logical_column->time_unit() ==
                    parquet::LogicalType::TimeUnit::MICROS ||
                time_logical_column->time_unit() ==
                    parquet::LogicalType::TimeUnit::NANOS);
          CHECK(parquet_column->physical_type() == parquet::Type::INT64);
          return create_parquet_time_encoder_with_types<int64_t, int64_t, int32_t>(
              omnisci_column, parquet_column, buffer);
        }
      } else {
        if (time_logical_column->time_unit() == parquet::LogicalType::TimeUnit::MILLIS) {
          CHECK(parquet_column->physical_type() == parquet::Type::INT32);
          return create_parquet_time_encoder_with_types<int32_t, int32_t, int32_t>(
              omnisci_column, parquet_column, buffer);
        } else {
          CHECK(time_logical_column->time_unit() ==
                    parquet::LogicalType::TimeUnit::MICROS ||
                time_logical_column->time_unit() ==
                    parquet::LogicalType::TimeUnit::NANOS);
          CHECK(parquet_column->physical_type() == parquet::Type::INT64);
          return create_parquet_time_encoder_with_types<int32_t, int64_t, int32_t>(
              omnisci_column, parquet_column, buffer);
        }
      }
    } else {
      UNREACHABLE();
    }
  }
  return {};
}

Here is the caller graph for this function:

template<typename V , typename T , typename NullType >

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_time_encoder_with_types	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer
	)

Definition at line 615 of file LazyParquetChunkLoader.cpp.

References UNREACHABLE.

                            {
  if (auto time_logical_type = dynamic_cast<const parquet::TimeLogicalType*>(
          parquet_column->logical_type().get())) {
    switch (time_logical_type->time_unit()) {
      case parquet::LogicalType::TimeUnit::MILLIS:
        return std::make_shared<ParquetTimeEncoder<V, T, 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      case parquet::LogicalType::TimeUnit::MICROS:
        return std::make_shared<ParquetTimeEncoder<V, T, 1000L * 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      case parquet::LogicalType::TimeUnit::NANOS:
        return std::make_shared<
            ParquetTimeEncoder<V, T, 1000L * 1000L * 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      default:
        UNREACHABLE();
    }
  } else {
    UNREACHABLE();
  }
  return {};
}

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_timestamp_encoder	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer,
		const bool	is_metadata_scan_or_for_import
	)

Definition at line 556 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfo::get_precision(), kENCODING_FIXED, kENCODING_NONE, and UNREACHABLE.

Referenced by create_parquet_encoder().

                                               {
  auto column_type = omnisci_column->columnType;
  auto precision = column_type.get_precision();
  if (parquet_column->logical_type()->is_timestamp()) {
    if (column_type.get_compression() == kENCODING_NONE) {
      if (precision == 0) {
        return create_parquet_timestamp_encoder_with_types<int64_t, int64_t, int64_t>(
            omnisci_column, parquet_column, buffer);
      } else {
        return std::make_shared<ParquetFixedLengthEncoder<int64_t, int64_t, int64_t>>(
            buffer, omnisci_column, parquet_column);
      }
    } else if (column_type.get_compression() == kENCODING_FIXED) {
      CHECK(column_type.get_comp_param() == 32);
      if (is_metadata_scan_or_for_import) {
        return create_parquet_timestamp_encoder_with_types<int64_t, int64_t, int32_t>(
            omnisci_column, parquet_column, buffer);
      } else {
        return create_parquet_timestamp_encoder_with_types<int32_t, int64_t, int32_t>(
            omnisci_column, parquet_column, buffer);
      }
    }
  } else if (parquet_column->logical_type()->is_none() && column_type.is_timestamp()) {
    if (parquet_column->physical_type() == parquet::Type::INT32) {
      CHECK(column_type.get_compression() == kENCODING_FIXED &&
            column_type.get_comp_param() == 32);
      if (is_metadata_scan_or_for_import) {
        return std::make_shared<ParquetFixedLengthEncoder<int64_t, int32_t, int32_t>>(
            buffer, omnisci_column, parquet_column);
      } else {
        return std::make_shared<ParquetFixedLengthEncoder<int32_t, int32_t, int32_t>>(
            buffer, omnisci_column, parquet_column);
      }
    } else if (parquet_column->physical_type() == parquet::Type::INT64) {
      if (column_type.get_compression() == kENCODING_NONE) {
        return std::make_shared<ParquetFixedLengthEncoder<int64_t, int64_t, int64_t>>(
            buffer, omnisci_column, parquet_column);
      } else if (column_type.get_compression() == kENCODING_FIXED) {
        CHECK(column_type.get_comp_param() == 32);
        if (is_metadata_scan_or_for_import) {
          return std::make_shared<ParquetFixedLengthEncoder<int64_t, int64_t, int32_t>>(
              buffer, omnisci_column, parquet_column);
        } else {
          return std::make_shared<ParquetFixedLengthEncoder<int32_t, int64_t, int32_t>>(
              buffer, omnisci_column, parquet_column);
        }
      }
    } else {
      UNREACHABLE();
    }
  }
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V , typename T , typename NullType >

std::shared_ptr<ParquetEncoder> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_timestamp_encoder_with_types	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		AbstractBuffer *	buffer
	)

Definition at line 482 of file LazyParquetChunkLoader.cpp.

References UNREACHABLE.

                            {
  if (auto timestamp_logical_type = dynamic_cast<const parquet::TimestampLogicalType*>(
          parquet_column->logical_type().get())) {
    switch (timestamp_logical_type->time_unit()) {
      case parquet::LogicalType::TimeUnit::MILLIS:
        return std::make_shared<ParquetTimestampEncoder<V, T, 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      case parquet::LogicalType::TimeUnit::MICROS:
        return std::make_shared<ParquetTimestampEncoder<V, T, 1000L * 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      case parquet::LogicalType::TimeUnit::NANOS:
        return std::make_shared<
            ParquetTimestampEncoder<V, T, 1000L * 1000L * 1000L, NullType>>(
            buffer, omnisci_column, parquet_column);
      default:
        UNREACHABLE();
    }
  } else {
    UNREACHABLE();
  }
  return {};
}

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_microsecond_precision

(

const ColumnDescriptor *

omnisci_column

)

Definition at line 1333 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::get_dimension().

Referenced by suggest_timestamp_mapping(), and validate_timestamp_mapping().

                                                                      {
  return omnisci_column->columnType.get_dimension() == 6;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_microsecond_precision

(

const parquet::TimestampLogicalType *

timestamp_logical_column

)

Definition at line 1337 of file LazyParquetChunkLoader.cpp.

                                                                 {
  return timestamp_logical_column->time_unit() == parquet::LogicalType::TimeUnit::MICROS;
}

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_millisecond_precision

(

const ColumnDescriptor *

omnisci_column

)

Definition at line 1342 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::get_dimension().

Referenced by suggest_timestamp_mapping(), and validate_timestamp_mapping().

                                                                      {
  return omnisci_column->columnType.get_dimension() == 3;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_millisecond_precision

(

const parquet::TimestampLogicalType *

timestamp_logical_column

)

Definition at line 1346 of file LazyParquetChunkLoader.cpp.

                                                                 {
  return timestamp_logical_column->time_unit() == parquet::LogicalType::TimeUnit::MILLIS;
}

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_nanosecond_precision

(

const ColumnDescriptor *

omnisci_column

)

Definition at line 1324 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::get_dimension().

Referenced by suggest_timestamp_mapping(), and validate_timestamp_mapping().

                                                                     {
  return omnisci_column->columnType.get_dimension() == 9;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_nanosecond_precision

(

const parquet::TimestampLogicalType *

timestamp_logical_column

)

Definition at line 1328 of file LazyParquetChunkLoader.cpp.

                                                                 {
  return timestamp_logical_column->time_unit() == parquet::LogicalType::TimeUnit::NANOS;
}

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_valid_parquet_list_column

(

const parquet::ColumnDescriptor *

parquet_column

)

Detect a valid list parquet column.

Parameters

parquet_column

- the parquet column descriptor of the column to detect

Returns

true if it is a valid parquet list column

Note: the notion of a valid parquet list column is adapted from the parquet schema specification for logical type definitions:

<list-repetition> group <name> (LIST) { repeated group list { <element-repetition> <element-type> element; } }

Testing has shown that there are small deviations from this specification in at least one library– pyarrow– where the innermost schema node is named "item" as opposed to "element".

The following is also true of the schema definition.

• The outer-most level must be a group annotated with LIST that contains a single field named list. The repetition of this level must be either optional or required and determines whether the list is nullable.
• The middle level, named list, must be a repeated group with a single field named element.
• The element field encodes the list's element type and repetition. Element repetition must be required or optional.

FSI further restricts lists to be defined only at the top level, meaning directly below the root schema node.

Definition at line 102 of file LazyParquetChunkLoader.cpp.

Referenced by create_parquet_array_encoder(), set_definition_levels_for_zero_max_definition_level_case(), foreign_storage::LazyParquetChunkLoader::suggestColumnMapping(), validate_allowed_mapping(), validate_column_mapping_and_row_group_metadata(), validate_list_column_metadata_statistics(), and validate_max_repetition_and_definition_level().

                                                                                 {
  const parquet::schema::Node* node = parquet_column->schema_node().get();
  if ((node->name() != "element" && node->name() != "item") ||
      !(node->is_required() ||
        node->is_optional())) {  // ensure first innermost node is named "element"
                                 // which is required by the parquet specification;
                                 // however testing shows that pyarrow generates this
                                 // column with the name of "item"
                                 // this field must be either required or optional
    return false;
  }
  node = node->parent();
  if (!node) {  // required nested structure
    return false;
  }
  if (node->name() != "list" || !node->is_repeated() ||
      !node->is_group()) {  // ensure second innermost node is named "list" which is
                            // a repeated group; this is
                            // required by the parquet specification
    return false;
  }
  node = node->parent();
  if (!node) {  // required nested structure
    return false;
  }
  if (!node->logical_type()->is_list() ||
      !(node->is_optional() ||
        node->is_required())) {  // ensure third outermost node has logical type LIST
                                 // which is either optional or required; this is required
                                 // by the parquet specification
    return false;
  }
  node =
      node->parent();  // this must now be the root node of schema which is required by
                       // FSI (lists can not be embedded into a deeper nested structure)
  if (!node) {         // required nested structure
    return false;
  }
  node = node->parent();
  if (node) {  // implies the previous node was not the root node
    return false;
  }
  return true;
}

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::is_valid_parquet_string

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 60 of file LazyParquetChunkLoader.cpp.

Referenced by create_parquet_geospatial_encoder(), create_parquet_string_encoder(), suggest_column_scalar_type(), suggest_string_mapping(), validate_geospatial_mapping(), and validate_string_mapping().

                                                                            {
  return (parquet_column->logical_type()->is_none() &&
          parquet_column->physical_type() == parquet::Type::BYTE_ARRAY) ||
         parquet_column->logical_type()->is_string();
}

Here is the caller graph for this function:

std::list<RowGroupMetadata> foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::metadata_scan_rowgroup_interval	(	const std::map< int, std::shared_ptr< ParquetEncoder >> &	encoder_map,
		const RowGroupInterval &	row_group_interval,
		const ReaderPtr &	reader,
		const ForeignTableSchema &	schema
	)

Definition at line 1719 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnId, foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, foreign_storage::ForeignTableSchema::getColumnDescriptor(), foreign_storage::ForeignTableSchema::getLogicalAndPhysicalColumns(), foreign_storage::ForeignTableSchema::getLogicalColumn(), foreign_storage::ForeignTableSchema::getParquetColumnIndex(), and foreign_storage::RowGroupInterval::start_index.

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan().

                                      {
  std::list<RowGroupMetadata> row_group_metadata;
  auto column_interval =
      Interval<ColumnType>{schema.getLogicalAndPhysicalColumns().front()->columnId,
                           schema.getLogicalAndPhysicalColumns().back()->columnId};

  auto file_metadata = reader->parquet_reader()->metadata();
  for (int row_group = row_group_interval.start_index;
       row_group <= row_group_interval.end_index;
       ++row_group) {
    auto& row_group_metadata_item = row_group_metadata.emplace_back();
    row_group_metadata_item.row_group_index = row_group;
    row_group_metadata_item.file_path = row_group_interval.file_path;

    std::unique_ptr<parquet::RowGroupMetaData> group_metadata =
        file_metadata->RowGroup(row_group);

    for (int column_id = column_interval.start; column_id <= column_interval.end;
         column_id++) {
      const auto column_descriptor = schema.getColumnDescriptor(column_id);
      auto parquet_column_index = schema.getParquetColumnIndex(column_id);
      auto encoder_map_iter =
          encoder_map.find(schema.getLogicalColumn(column_id)->columnId);
      CHECK(encoder_map_iter != encoder_map.end());
      try {
        auto metadata = encoder_map_iter->second->getRowGroupMetadata(
            group_metadata.get(), parquet_column_index, column_descriptor->columnType);
        row_group_metadata_item.column_chunk_metadata.emplace_back(metadata);
      } catch (const std::exception& e) {
        std::stringstream error_message;
        error_message << e.what() << " in row group " << row_group << " of Parquet file '"
                      << row_group_interval.file_path << "'.";
        throw std::runtime_error(error_message.str());
      }
    }
  }
  return row_group_metadata;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::map<int, std::shared_ptr<ParquetEncoder> > foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::populate_encoder_map_for_import	(	const std::map< int, Chunk_NS::Chunk >	chunks,
		const ForeignTableSchema &	schema,
		const ReaderPtr &	reader,
		const std::map< int, StringDictionary * >	column_dictionaries,
		const int64_t	num_rows,
		const bool	geo_validate_geometry
	)

Definition at line 1762 of file LazyParquetChunkLoader.cpp.

References create_parquet_encoder_for_import(), shared::get_from_map(), foreign_storage::ForeignTableSchema::getColumnDescriptor(), and foreign_storage::ForeignTableSchema::getParquetColumnIndex().

Referenced by foreign_storage::LazyParquetChunkLoader::loadRowGroups().

                                      {
  std::map<int, std::shared_ptr<ParquetEncoder>> encoder_map;
  auto file_metadata = reader->parquet_reader()->metadata();
  for (auto& [column_id, chunk] : chunks) {
    const auto column_descriptor = schema.getColumnDescriptor(column_id);
    if (column_descriptor->isGeoPhyCol) {  // skip physical columns
      continue;
    }
    auto parquet_column_descriptor =
        file_metadata->schema()->Column(schema.getParquetColumnIndex(column_id));
    auto find_it = column_dictionaries.find(column_id);
    StringDictionary* dictionary =
        (find_it == column_dictionaries.end() ? nullptr : find_it->second);
    std::list<Chunk_NS::Chunk> chunks_for_import;
    chunks_for_import.push_back(chunk);
    if (column_descriptor->columnType.is_geometry()) {
      for (int i = 0; i < column_descriptor->columnType.get_physical_cols(); ++i) {
        chunks_for_import.push_back(chunks.at(column_id + i + 1));
      }
    }
    encoder_map[column_id] = create_parquet_encoder_for_import(chunks_for_import,
                                                               column_descriptor,
                                                               parquet_column_descriptor,
                                                               dictionary,
                                                               geo_validate_geometry);

    // reserve space in buffer when num-elements known ahead of time for types
    // of known size (for example dictionary encoded strings)
    auto encoder = shared::get_from_map(encoder_map, column_id);
    if (auto inplace_encoder = dynamic_cast<ParquetInPlaceEncoder*>(encoder.get())) {
      inplace_encoder->reserve(num_rows);
    }
  }
  return encoder_map;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::map<int, std::shared_ptr<ParquetEncoder> > foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::populate_encoder_map_for_metadata_scan	(	const Interval< ColumnType > &	column_interval,
		const ForeignTableSchema &	schema,
		const ReaderPtr &	reader,
		const bool	do_metadata_stats_validation,
		const bool	geo_validate_geometry
	)

Definition at line 1804 of file LazyParquetChunkLoader.cpp.

References create_parquet_encoder_for_metadata_scan(), foreign_storage::Interval< T >::end, shared::get_from_map(), foreign_storage::ForeignTableSchema::getColumnDescriptor(), foreign_storage::ForeignTableSchema::getParquetColumnIndex(), and foreign_storage::Interval< T >::start.

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan().

                                      {
  std::map<int, std::shared_ptr<ParquetEncoder>> encoder_map;
  auto file_metadata = reader->parquet_reader()->metadata();
  for (int column_id = column_interval.start; column_id <= column_interval.end;
       column_id++) {
    const auto column_descriptor = schema.getColumnDescriptor(column_id);
    auto parquet_column_descriptor =
        file_metadata->schema()->Column(schema.getParquetColumnIndex(column_id));
    encoder_map[column_id] = create_parquet_encoder_for_metadata_scan(
        column_descriptor, parquet_column_descriptor, geo_validate_geometry);
    if (!do_metadata_stats_validation) {
      shared::get_from_map(encoder_map, column_id)->disableMetadataStatsValidation();
    }
    column_id += column_descriptor->columnType.get_physical_cols();
  }
  return encoder_map;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::resize_values_buffer	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column,
		std::vector< int8_t > &	values
	)

Definition at line 1171 of file LazyParquetChunkLoader.cpp.

References foreign_storage::LazyParquetChunkLoader::batch_reader_num_elements, ColumnDescriptor::columnType, and SQLTypeInfo::get_size().

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::ParquetRowGroupReader::readAndValidateRowGroup().

                                                     {
  auto max_type_byte_size =
      std::max(omnisci_column->columnType.get_size(),
               parquet::GetTypeByteSize(parquet_column->physical_type()));
  size_t values_size =
      LazyParquetChunkLoader::batch_reader_num_elements * max_type_byte_size;
  values.resize(values_size);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::set_definition_levels_for_zero_max_definition_level_case	(	const parquet::ColumnDescriptor *	parquet_column_descriptor,
		std::vector< int16_t > &	def_levels
	)

This function sets the definition levels to 1 for all read values in the case of required scalar/flat columns. The definition level of one informs all subsequent calls to parquet encoders to treat the read data as not null.

Definition at line 1114 of file LazyParquetChunkLoader.cpp.

References is_valid_parquet_list_column().

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups().

                                    {
  if (!is_valid_parquet_list_column(parquet_column_descriptor) &&
      parquet_column_descriptor->max_definition_level() == 0) {
    if (!parquet_column_descriptor->schema_node()->is_required()) {
      throw std::runtime_error(
          "Unsupported parquet column detected. Column '" +
          parquet_column_descriptor->path()->ToDotString() +
          "' detected to have max definition level of 0 but is optional.");
    }
    def_levels.assign(def_levels.size(), 1);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_boolean_type_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1360 of file LazyParquetChunkLoader.cpp.

References kBOOLEAN, kENCODING_NONE, SQLTypeInfo::set_compression(), SQLTypeInfo::set_fixed_size(), SQLTypeInfo::set_type(), and run_benchmark_import::type.

Referenced by suggest_column_scalar_type().

                                                   {
  SQLTypeInfo type;
  type.set_compression(kENCODING_NONE);
  type.set_type(kBOOLEAN);
  type.set_fixed_size();
  return type;
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_column_scalar_type

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1568 of file LazyParquetChunkLoader.cpp.

References is_valid_parquet_string(), suggest_boolean_type_mapping(), suggest_date_mapping(), suggest_decimal_mapping(), suggest_floating_point_mapping(), suggest_integral_mapping(), suggest_string_mapping(), suggest_time_mapping(), and suggest_timestamp_mapping().

Referenced by foreign_storage::LazyParquetChunkLoader::suggestColumnMapping().

                                                                                      {
  // decimal case
  if (parquet_column->logical_type()->is_decimal()) {
    return suggest_decimal_mapping(parquet_column);
  }
  // float case
  if (parquet_column->logical_type()->is_none() &&
      (parquet_column->physical_type() == parquet::Type::FLOAT ||
       parquet_column->physical_type() == parquet::Type::DOUBLE)) {
    return suggest_floating_point_mapping(parquet_column);
  }
  // integral case
  if ((parquet_column->logical_type()->is_none() &&
       (parquet_column->physical_type() == parquet::Type::INT32 ||
        parquet_column->physical_type() == parquet::Type::INT64)) ||
      parquet_column->logical_type()->is_int()) {
    return suggest_integral_mapping(parquet_column);
  }
  // boolean case
  if (parquet_column->logical_type()->is_none() &&
      parquet_column->physical_type() == parquet::Type::BOOLEAN) {
    return suggest_boolean_type_mapping(parquet_column);
  }
  // timestamp case
  if (parquet_column->logical_type()->is_timestamp()) {
    return suggest_timestamp_mapping(parquet_column);
  }
  // time case
  if (parquet_column->logical_type()->is_time()) {
    return suggest_time_mapping(parquet_column);
  }
  // date case
  if (parquet_column->logical_type()->is_date()) {
    return suggest_date_mapping(parquet_column);
  }
  // string case
  if (is_valid_parquet_string(parquet_column)) {
    return suggest_string_mapping(parquet_column);
  }

  throw ForeignStorageException("Unsupported data type detected for column: " +
                                parquet_column->ToString());
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_date_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1464 of file LazyParquetChunkLoader.cpp.

References CHECK, kDATE, kENCODING_NONE, and run_benchmark_import::type.

Referenced by suggest_column_scalar_type().

                                                                                {
  CHECK(parquet_column->logical_type()->is_date());
  SQLTypeInfo type;
  type.set_type(kDATE);
  type.set_compression(kENCODING_NONE);
  type.set_fixed_size();
  return type;
}

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_decimal_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1196 of file LazyParquetChunkLoader.cpp.

References kDECIMAL, kENCODING_NONE, sql_constants::kMaxNumericPrecision, SQLTypeInfo::scale, SQLTypeInfo::set_compression(), SQLTypeInfo::set_fixed_size(), SQLTypeInfo::set_precision(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_type(), to_string(), run_benchmark_import::type, and UNREACHABLE.

Referenced by suggest_column_scalar_type().

                                                                                   {
  if (auto decimal_logical_column = dynamic_cast<const parquet::DecimalLogicalType*>(
          parquet_column->logical_type().get())) {
    auto parquet_precision = decimal_logical_column->precision();
    auto parquet_scale = decimal_logical_column->scale();
    if (parquet_precision > sql_constants::kMaxNumericPrecision) {
      throw ForeignStorageException(
          "Parquet column \"" + parquet_column->ToString() +
          "\" has decimal precision of " + std::to_string(parquet_precision) +
          " which is too high to import, maximum precision supported is " +
          std::to_string(sql_constants::kMaxNumericPrecision) + ".");
    }
    SQLTypeInfo type;
    type.set_type(kDECIMAL);
    type.set_compression(kENCODING_NONE);
    type.set_precision(parquet_precision);
    type.set_scale(parquet_scale);
    type.set_fixed_size();
    return type;
  }
  UNREACHABLE()
      << " a Parquet column's decimal logical type failed to be read appropriately";
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_floating_point_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1237 of file LazyParquetChunkLoader.cpp.

References kDOUBLE, kENCODING_NONE, kFLOAT, SQLTypeInfo::set_compression(), SQLTypeInfo::set_fixed_size(), SQLTypeInfo::set_type(), run_benchmark_import::type, and UNREACHABLE.

Referenced by suggest_column_scalar_type().

                                                   {
  SQLTypeInfo type;
  if (parquet_column->physical_type() == parquet::Type::FLOAT) {
    type.set_type(kFLOAT);
  } else if (parquet_column->physical_type() == parquet::Type::DOUBLE) {
    type.set_type(kDOUBLE);
  } else {
    UNREACHABLE();
  }
  type.set_compression(kENCODING_NONE);
  type.set_fixed_size();
  return type;
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_integral_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1279 of file LazyParquetChunkLoader.cpp.

References CHECK, kBIGINT, kENCODING_NONE, kINT, kSMALLINT, kTINYINT, SQLTypeInfo::set_compression(), SQLTypeInfo::set_fixed_size(), SQLTypeInfo::set_type(), run_benchmark_import::type, and within_range().

Referenced by suggest_column_scalar_type().

                                                                                    {
  SQLTypeInfo type;
  type.set_compression(kENCODING_NONE);
  if (auto int_logical_column = dynamic_cast<const parquet::IntLogicalType*>(
          parquet_column->logical_type().get())) {
    auto bit_width = int_logical_column->bit_width();
    if (!int_logical_column->is_signed()) {
      if (within_range(33, 64, bit_width)) {
        throw ForeignStorageException(
            "Unsigned integer column \"" + parquet_column->path()->ToDotString() +
            "\" in Parquet file with 64 bit-width has no supported type for ingestion "
            "that will not result in data loss");
      } else if (within_range(17, 32, bit_width)) {
        type.set_type(kBIGINT);
      } else if (within_range(9, 16, bit_width)) {
        type.set_type(kINT);
      } else if (within_range(0, 8, bit_width)) {
        type.set_type(kSMALLINT);
      }
    } else {
      if (within_range(33, 64, bit_width)) {
        type.set_type(kBIGINT);
      } else if (within_range(17, 32, bit_width)) {
        type.set_type(kINT);
      } else if (within_range(9, 16, bit_width)) {
        type.set_type(kSMALLINT);
      } else if (within_range(0, 8, bit_width)) {
        type.set_type(kTINYINT);
      }
    }
    type.set_fixed_size();
    return type;
  }

  CHECK(parquet_column->logical_type()->is_none());
  if (parquet_column->physical_type() == parquet::Type::INT32) {
    type.set_type(kINT);
  } else {
    CHECK(parquet_column->physical_type() == parquet::Type::INT64);
    type.set_type(kBIGINT);
  }
  type.set_fixed_size();
  return type;
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_string_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1481 of file LazyParquetChunkLoader.cpp.

References CHECK, is_valid_parquet_string(), kENCODING_DICT, kTEXT, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_fixed_size(), SQLTypeInfo::set_type(), and run_benchmark_import::type.

Referenced by suggest_column_scalar_type().

                                                                                  {
  CHECK(is_valid_parquet_string(parquet_column));
  SQLTypeInfo type;
  type.set_type(kTEXT);
  type.set_compression(kENCODING_DICT);
  type.set_comp_param(0);  // `comp_param` is expected either to be zero or
                           // equal to a string dictionary id in some code
                           // paths, since we don't have a string dictionary we
                           // set this to zero
  type.set_fixed_size();
  return type;
}

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_time_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1438 of file LazyParquetChunkLoader.cpp.

References CHECK, kENCODING_NONE, kTIME, and run_benchmark_import::type.

Referenced by suggest_column_scalar_type().

                                                                                {
  CHECK(parquet_column->logical_type()->is_time());
  SQLTypeInfo type;
  type.set_type(kTIME);
  type.set_compression(kENCODING_NONE);
  type.set_fixed_size();
  return type;
}

Here is the caller graph for this function:

SQLTypeInfo foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::suggest_timestamp_mapping

(

const parquet::ColumnDescriptor *

parquet_column

)

Definition at line 1404 of file LazyParquetChunkLoader.cpp.

References is_microsecond_precision(), is_millisecond_precision(), is_nanosecond_precision(), kENCODING_NONE, kTIMESTAMP, SQLTypeInfo::set_compression(), SQLTypeInfo::set_fixed_size(), SQLTypeInfo::set_precision(), SQLTypeInfo::set_type(), run_benchmark_import::type, and UNREACHABLE.

Referenced by suggest_column_scalar_type().

                                                                                     {
  if (auto timestamp_logical_column = dynamic_cast<const parquet::TimestampLogicalType*>(
          parquet_column->logical_type().get())) {
    SQLTypeInfo type;
    type.set_type(kTIMESTAMP);
    type.set_compression(kENCODING_NONE);
    if (is_nanosecond_precision(timestamp_logical_column)) {
      type.set_precision(9);
    } else if (is_microsecond_precision(timestamp_logical_column)) {
      type.set_precision(6);
    } else if (is_millisecond_precision(timestamp_logical_column)) {
      type.set_precision(3);
    }
    type.set_fixed_size();
    return type;
  }
  UNREACHABLE();
  return {};
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::throw_missing_metadata_error	(	const int	row_group_index,
		const int	column_index,
		const std::string &	file_path
	)

Definition at line 1622 of file LazyParquetChunkLoader.cpp.

References to_string().

Referenced by validate_column_mapping_and_row_group_metadata().

                                                              {
  throw std::runtime_error{
      "Statistics metadata is required for all row groups. Metadata is missing for "
      "row group index: " +
      std::to_string(row_group_index) +
      ", column index: " + std::to_string(column_index) + ", file path: " + file_path};
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::throw_row_group_larger_than_fragment_size_error	(	const MaxRowGroupSizeStats	max_row_group_stats,
		const int	fragment_size
	)

Definition at line 1638 of file LazyParquetChunkLoader.cpp.

References foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::MaxRowGroupSizeStats::file_path, foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::MaxRowGroupSizeStats::max_row_group_index, foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::MaxRowGroupSizeStats::max_row_group_size, and to_string().

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan().

                             {
  auto metadata_scan_exception = MetadataScanInfeasibleFragmentSizeException{
      "Parquet file has a row group size that is larger than the fragment size. "
      "Please set the table fragment size to a number that is larger than the "
      "row group size. Row group index: " +
      std::to_string(max_row_group_stats.max_row_group_index) +
      ", row group size: " + std::to_string(max_row_group_stats.max_row_group_size) +
      ", fragment size: " + std::to_string(fragment_size) +
      ", file path: " + max_row_group_stats.file_path};
  metadata_scan_exception.min_feasible_fragment_size_ =
      max_row_group_stats.max_row_group_size;
  throw metadata_scan_exception;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_allowed_mapping	(	const parquet::ColumnDescriptor *	parquet_column,
		const ColumnDescriptor *	omnisci_column
	)

Definition at line 1526 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, foreign_storage::get_sub_type_column_descriptor(), SQLTypeInfo::get_type_name(), SQLTypeInfo::is_array(), is_valid_parquet_list_column(), foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported(), LOG, run_benchmark_import::type, validate_max_repetition_and_definition_level(), and logger::WARNING.

Referenced by foreign_storage::LazyParquetChunkLoader::loadRowGroups(), and validate_column_mapping_and_row_group_metadata().

                                                                      {
  validate_max_repetition_and_definition_level(omnisci_column, parquet_column);
  bool allowed_type = false;
  if (omnisci_column->columnType.is_array()) {
    if (is_valid_parquet_list_column(parquet_column)) {
      auto omnisci_column_sub_type_column =
          get_sub_type_column_descriptor(omnisci_column);
      allowed_type = LazyParquetChunkLoader::isColumnMappingSupported(
          omnisci_column_sub_type_column.get(), parquet_column);
    }
  } else {
    allowed_type =
        LazyParquetChunkLoader::isColumnMappingSupported(omnisci_column, parquet_column);
  }
  if (!allowed_type) {
    auto logical_type = parquet_column->logical_type();
    if (logical_type->is_timestamp()) {
      auto timestamp_type =
          dynamic_cast<const parquet::TimestampLogicalType*>(logical_type.get());
      CHECK(timestamp_type);

      if (!timestamp_type->is_adjusted_to_utc()) {
        LOG(WARNING) << "Non-UTC timezone specified in Parquet file for column \""
                     << omnisci_column->columnName
                     << "\". Only UTC timezone is currently supported.";
      }
    }
    std::string parquet_type;
    parquet::Type::type physical_type = parquet_column->physical_type();
    if (parquet_column->logical_type()->is_none()) {
      parquet_type = parquet::TypeToString(physical_type);
    } else {
      parquet_type = logical_type->ToString();
    }
    std::string omnisci_type = omnisci_column->columnType.get_type_name();
    throw std::runtime_error{"Conversion from Parquet type \"" + parquet_type +
                             "\" to HeavyDB type \"" + omnisci_type +
                             "\" is not allowed. Please use an appropriate column type."};
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

MaxRowGroupSizeStats foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_column_mapping_and_row_group_metadata	(	const std::shared_ptr< parquet::FileMetaData > &	file_metadata,
		const std::string &	file_path,
		const ForeignTableSchema &	schema,
		const bool	do_metadata_stats_validation
	)

Definition at line 1654 of file LazyParquetChunkLoader.cpp.

References foreign_storage::ForeignTableSchema::getLogicalColumns(), is_valid_parquet_list_column(), report::stats, throw_missing_metadata_error(), and validate_allowed_mapping().

Referenced by validate_parquet_metadata().

                                             {
  auto column_it = schema.getLogicalColumns().begin();
  MaxRowGroupSizeStats max_row_group_stats{0, 0};
  for (int i = 0; i < file_metadata->num_columns(); ++i, ++column_it) {
    const parquet::ColumnDescriptor* descr = file_metadata->schema()->Column(i);
    try {
      validate_allowed_mapping(descr, *column_it);
    } catch (std::runtime_error& e) {
      std::stringstream error_message;
      error_message << e.what() << " Parquet column: " << descr->path()->ToDotString()
                    << ", HeavyDB column: " << (*column_it)->columnName
                    << ", Parquet file: " << file_path << ".";
      throw std::runtime_error(error_message.str());
    }

    for (int r = 0; r < file_metadata->num_row_groups(); ++r) {
      auto group_metadata = file_metadata->RowGroup(r);
      auto num_rows = group_metadata->num_rows();
      if (num_rows == 0) {
        continue;
      } else if (num_rows > max_row_group_stats.max_row_group_size) {
        max_row_group_stats.max_row_group_size = num_rows;
        max_row_group_stats.max_row_group_index = r;
        max_row_group_stats.file_path = file_path;
      }

      if (do_metadata_stats_validation) {
        auto column_chunk = group_metadata->ColumnChunk(i);
        bool contains_metadata = column_chunk->is_stats_set();
        if (contains_metadata) {
          auto stats = column_chunk->statistics();
          bool is_all_nulls = stats->null_count() == column_chunk->num_values();
          bool is_list = is_valid_parquet_list_column(file_metadata->schema()->Column(i));
          // Given a list, it is possible it has no min or max if it is comprised
          // only of empty lists & nulls. This can not be detected by comparing
          // the null count; therefore we afford list types the benefit of the
          // doubt in this situation.
          if (!(stats->HasMinMax() || is_all_nulls || is_list)) {
            contains_metadata = false;
          }
        }

        if (!contains_metadata) {
          throw_missing_metadata_error(r, i, file_path);
        }
      }
    }
  }
  return max_row_group_stats;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_date_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1447 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_type(), kDATE, kENCODING_DATE_IN_DAYS, and kENCODING_NONE.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                          {
  if (!(omnisci_column->columnType.get_type() == kDATE &&
        ((omnisci_column->columnType.get_compression() == kENCODING_DATE_IN_DAYS &&
          (omnisci_column->columnType.get_comp_param() ==
               0  // DATE ENCODING DAYS (32) specifies comp_param of 0
           || omnisci_column->columnType.get_comp_param() == 16)) ||
         omnisci_column->columnType.get_compression() ==
             kENCODING_NONE  // for array types
         ))) {
    return false;
  }
  return parquet_column->logical_type()->is_date() ||
         parquet_column->logical_type()
             ->is_timestamp();  // to support TIMESTAMP -> DATE coercion
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_decimal_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1182 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_precision(), SQLTypeInfo::get_scale(), SQLTypeInfo::is_decimal(), kENCODING_FIXED, and kENCODING_NONE.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                             {
  if (auto decimal_logical_column = dynamic_cast<const parquet::DecimalLogicalType*>(
          parquet_column->logical_type().get())) {
    return omnisci_column->columnType.get_precision() ==
               decimal_logical_column->precision() &&
           omnisci_column->columnType.get_scale() == decimal_logical_column->scale() &&
           omnisci_column->columnType.is_decimal() &&
           (omnisci_column->columnType.get_compression() == kENCODING_NONE ||
            omnisci_column->columnType.get_compression() == kENCODING_FIXED);
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema	(	const parquet::arrow::FileReader *	reference_file_reader,
		const parquet::arrow::FileReader *	new_file_reader,
		const std::string &	reference_file_path,
		const std::string &	new_file_path
	)

Definition at line 1500 of file LazyParquetChunkLoader.cpp.

References foreign_storage::get_column_descriptor(), to_string(), and foreign_storage::validate_equal_column_descriptor().

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan(), and foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                           {
  const auto reference_num_columns =
      reference_file_reader->parquet_reader()->metadata()->num_columns();
  const auto new_num_columns =
      new_file_reader->parquet_reader()->metadata()->num_columns();
  if (reference_num_columns != new_num_columns) {
    throw std::runtime_error{"Parquet file \"" + new_file_path +
                             "\" has a different schema. Please ensure that all Parquet "
                             "files use the same schema. Reference Parquet file: \"" +
                             reference_file_path + "\" has " +
                             std::to_string(reference_num_columns) +
                             " columns. New Parquet file \"" + new_file_path + "\" has " +
                             std::to_string(new_num_columns) + " columns."};
  }

  for (int i = 0; i < reference_num_columns; i++) {
    validate_equal_column_descriptor(get_column_descriptor(reference_file_reader, i),
                                     get_column_descriptor(new_file_reader, i),
                                     reference_file_path,
                                     new_file_path);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_floating_point_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1221 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_type(), SQLTypeInfo::is_fp(), kENCODING_NONE, and kFLOAT.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                                    {
  if (!omnisci_column->columnType.is_fp()) {
    return false;
  }
  // check if mapping is a valid coerced or non-coerced floating point mapping
  // with no annotation (floating point columns have no annotation in the
  // Parquet specification)
  if (omnisci_column->columnType.get_compression() == kENCODING_NONE) {
    return (parquet_column->physical_type() == parquet::Type::DOUBLE) ||
           (parquet_column->physical_type() == parquet::Type::FLOAT &&
            omnisci_column->columnType.get_type() == kFLOAT);
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_geospatial_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1494 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::is_geometry(), and is_valid_parquet_string().

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                                {
  return is_valid_parquet_string(parquet_column) &&
         omnisci_column->columnType.is_geometry();
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_integral_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1252 of file LazyParquetChunkLoader.cpp.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::is_integer(), kENCODING_FIXED, and kENCODING_NONE.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                              {
  if (!omnisci_column->columnType.is_integer()) {
    return false;
  }
  if (auto int_logical_column = dynamic_cast<const parquet::IntLogicalType*>(
          parquet_column->logical_type().get())) {
    CHECK(omnisci_column->columnType.get_compression() == kENCODING_NONE ||
          omnisci_column->columnType.get_compression() == kENCODING_FIXED);
    const int bits_per_byte = 8;
    // unsigned types are permitted to map to a wider integral type in order to avoid
    // precision loss
    const int bit_widening_factor = int_logical_column->is_signed() ? 1 : 2;
    return omnisci_column->columnType.get_size() * bits_per_byte <=
           int_logical_column->bit_width() * bit_widening_factor;
  }
  // check if mapping is a valid coerced or non-coerced integral mapping with no
  // annotation
  if ((omnisci_column->columnType.get_compression() == kENCODING_NONE ||
       omnisci_column->columnType.get_compression() == kENCODING_FIXED)) {
    return (parquet_column->physical_type() == parquet::Type::INT64) ||
           (parquet_column->physical_type() == parquet::Type::INT32 &&
            omnisci_column->columnType.get_size() <= 4);
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_list_column_metadata_statistics	(	const parquet::ParquetFileReader *	reader,
		const int	row_group_index,
		const int	column_index,
		const int16_t *	def_levels,
		const int64_t	num_levels,
		const parquet::ColumnDescriptor *	parquet_column_descriptor
	)

Definition at line 1078 of file LazyParquetChunkLoader.cpp.

References is_valid_parquet_list_column(), report::stats, and foreign_storage::validate_and_get_column_metadata_statistics().

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::ParquetRowGroupReader::readAndValidateRowGroup().

                                                              {
  bool is_valid_parquet_list = is_valid_parquet_list_column(parquet_column_descriptor);
  if (!is_valid_parquet_list) {
    return;
  }
  std::unique_ptr<parquet::RowGroupMetaData> group_metadata =
      reader->metadata()->RowGroup(row_group_index);
  auto column_metadata = group_metadata->ColumnChunk(column_index);
  // In case of a empty row group do not validate
  if (group_metadata->num_rows() == 0) {
    return;
  }
  auto stats = validate_and_get_column_metadata_statistics(column_metadata.get());
  if (!stats->HasMinMax()) {
    auto find_it = std::find_if(def_levels,
                                def_levels + num_levels,
                                [](const int16_t def_level) { return def_level == 3; });
    if (find_it != def_levels + num_levels) {
      throw std::runtime_error(
          "No minimum and maximum statistic set in list column but non-null & non-empty "
          "array/value detected.");
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_max_repetition_and_definition_level	(	const ColumnDescriptor *	omnisci_column_descriptor,
		const parquet::ColumnDescriptor *	parquet_column_descriptor
	)

Definition at line 1129 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_array(), is_valid_parquet_list_column(), and to_string().

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), foreign_storage::LazyParquetChunkLoader::loadRowGroups(), and validate_allowed_mapping().

                                                              {
  bool is_valid_parquet_list = is_valid_parquet_list_column(parquet_column_descriptor);
  if (is_valid_parquet_list && !omnisci_column_descriptor->columnType.is_array()) {
    throw std::runtime_error(
        "Unsupported mapping detected. Column '" +
        parquet_column_descriptor->path()->ToDotString() +
        "' detected to be a parquet list but HeavyDB mapped column '" +
        omnisci_column_descriptor->columnName + "' is not an array.");
  }
  if (is_valid_parquet_list) {
    if (parquet_column_descriptor->max_repetition_level() != 1 ||
        parquet_column_descriptor->max_definition_level() != 3) {
      throw std::runtime_error(
          "Incorrect schema max repetition level detected in column '" +
          parquet_column_descriptor->path()->ToDotString() +
          "'. Expected a max repetition level of 1 and max definition level of 3 for "
          "list column but column has a max "
          "repetition level of " +
          std::to_string(parquet_column_descriptor->max_repetition_level()) +
          " and a max definition level of " +
          std::to_string(parquet_column_descriptor->max_definition_level()) + ".");
    }
  } else {
    if (parquet_column_descriptor->max_repetition_level() != 0 ||
        !(parquet_column_descriptor->max_definition_level() == 1 ||
          parquet_column_descriptor->max_definition_level() == 0)) {
      throw std::runtime_error(
          "Incorrect schema max repetition level detected in column '" +
          parquet_column_descriptor->path()->ToDotString() +
          "'. Expected a max repetition level of 0 and max definition level of 1 or 0 "
          "for "
          "flat column but column has a max "
          "repetition level of " +
          std::to_string(parquet_column_descriptor->max_repetition_level()) +
          " and a max definition level of " +
          std::to_string(parquet_column_descriptor->max_definition_level()) + ".");
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_none_type_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1351 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_type(), kBOOLEAN, and kENCODING_NONE.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                               {
  bool is_none_encoded_mapping =
      omnisci_column->columnType.get_compression() == kENCODING_NONE &&
      (parquet_column->physical_type() == parquet::Type::BOOLEAN &&
       omnisci_column->columnType.get_type() == kBOOLEAN);
  return parquet_column->logical_type()->is_none() && is_none_encoded_mapping;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_number_of_columns	(	const std::shared_ptr< parquet::FileMetaData > &	file_metadata,
		const std::string &	file_path,
		const ForeignTableSchema &	schema
	)

Definition at line 1612 of file LazyParquetChunkLoader.cpp.

References foreign_storage::ForeignTableSchema::numLogicalColumns(), and foreign_storage::throw_number_of_columns_mismatch_error().

Referenced by foreign_storage::LazyParquetChunkLoader::loadRowGroups(), and validate_parquet_metadata().

                                      {
  if (schema.numLogicalColumns() != file_metadata->num_columns()) {
    throw_number_of_columns_mismatch_error(
        schema.numLogicalColumns(), file_metadata->num_columns(), file_path);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

MaxRowGroupSizeStats foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_parquet_metadata	(	const std::shared_ptr< parquet::FileMetaData > &	file_metadata,
		const std::string &	file_path,
		const ForeignTableSchema &	schema,
		const bool	do_metadata_stats_validation
	)

Definition at line 1709 of file LazyParquetChunkLoader.cpp.

References validate_column_mapping_and_row_group_metadata(), and validate_number_of_columns().

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan().

                                             {
  validate_number_of_columns(file_metadata, file_path, schema);
  return validate_column_mapping_and_row_group_metadata(
      file_metadata, file_path, schema, do_metadata_stats_validation);
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_string_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1473 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), is_valid_parquet_string(), kENCODING_DICT, and kENCODING_NONE.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                            {
  return is_valid_parquet_string(parquet_column) &&
         omnisci_column->columnType.is_string() &&
         (omnisci_column->columnType.get_compression() == kENCODING_NONE ||
          omnisci_column->columnType.get_compression() == kENCODING_DICT);
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_time_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1424 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_type(), kENCODING_FIXED, kENCODING_NONE, and kTIME.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                          {
  if (!(omnisci_column->columnType.get_type() == kTIME &&
        (omnisci_column->columnType.get_compression() == kENCODING_NONE ||
         (omnisci_column->columnType.get_compression() == kENCODING_FIXED &&
          omnisci_column->columnType.get_comp_param() == 32)))) {
    return false;
  }
  if (parquet_column->logical_type()->is_time()) {
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_timestamp_mapping	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

Definition at line 1369 of file LazyParquetChunkLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_type(), is_microsecond_precision(), is_millisecond_precision(), is_nanosecond_precision(), kENCODING_FIXED, kENCODING_NONE, and kTIMESTAMP.

Referenced by foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported().

                                                                               {
  if (!(omnisci_column->columnType.get_type() == kTIMESTAMP &&
        ((omnisci_column->columnType.get_compression() == kENCODING_NONE) ||
         (omnisci_column->columnType.get_compression() == kENCODING_FIXED &&
          omnisci_column->columnType.get_comp_param() == 32)))) {
    return false;
  }
  // check the annotated case
  if (auto timestamp_logical_column = dynamic_cast<const parquet::TimestampLogicalType*>(
          parquet_column->logical_type().get())) {
    if (omnisci_column->columnType.get_compression() == kENCODING_NONE) {
      return omnisci_column->columnType.get_dimension() == 0 ||
             ((is_nanosecond_precision(omnisci_column) &&
               is_nanosecond_precision(timestamp_logical_column)) ||
              (is_microsecond_precision(omnisci_column) &&
               is_microsecond_precision(timestamp_logical_column)) ||
              (is_millisecond_precision(omnisci_column) &&
               is_millisecond_precision(timestamp_logical_column)));
    }
    if (omnisci_column->columnType.get_compression() == kENCODING_FIXED) {
      return omnisci_column->columnType.get_dimension() == 0;
    }
  }
  // check the unannotated case
  if (parquet_column->logical_type()->is_none() &&
      ((parquet_column->physical_type() == parquet::Type::INT32 &&
        omnisci_column->columnType.get_compression() == kENCODING_FIXED &&
        omnisci_column->columnType.get_comp_param() == 32) ||
       parquet_column->physical_type() == parquet::Type::INT64)) {
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::within_range	(	int64_t	lower_bound,
		int64_t	upper_bound,
		int64_t	value
	)

Definition at line 56 of file LazyParquetChunkLoader.cpp.

References gpu_enabled::upper_bound().

Referenced by suggest_integral_mapping().

                                                                           {
  return value >= lower_bound && value <= upper_bound;
}

Here is the call graph for this function:

Here is the caller graph for this function: