OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::ParquetRowGroupReader Class Reference
+ Collaboration diagram for foreign_storage::ParquetRowGroupReader:

Public Member Functions

 ParquetRowGroupReader (std::shared_ptr< parquet::ColumnReader > col_reader, const ColumnDescriptor *column_descriptor, const parquet::ColumnDescriptor *parquet_column_descriptor, ParquetEncoder *encoder, InvalidRowGroupIndices &invalid_indices, const int row_group_index, const int parquet_column_index, const parquet::ParquetFileReader *parquet_reader)
 
void readAndValidateRowGroup ()
 
void eraseInvalidRowGroupData ()
 

Private Attributes

std::shared_ptr
< parquet::ColumnReader > 
col_reader_
 
const ColumnDescriptorcolumn_descriptor_
 
const parquet::ColumnDescriptor * parquet_column_descriptor_
 
ParquetEncoderencoder_
 
ParquetImportEncoderimport_encoder
 
InvalidRowGroupIndicesinvalid_indices_
 
const int row_group_index_
 
const int parquet_column_index_
 
const parquet::ParquetFileReader * parquet_reader_
 

Detailed Description

Definition at line 1707 of file LazyParquetChunkLoader.cpp.

Constructor & Destructor Documentation

foreign_storage::ParquetRowGroupReader::ParquetRowGroupReader ( std::shared_ptr< parquet::ColumnReader >  col_reader,
const ColumnDescriptor column_descriptor,
const parquet::ColumnDescriptor *  parquet_column_descriptor,
ParquetEncoder encoder,
InvalidRowGroupIndices invalid_indices,
const int  row_group_index,
const int  parquet_column_index,
const parquet::ParquetFileReader *  parquet_reader 
)
inline

Definition at line 1709 of file LazyParquetChunkLoader.cpp.

References CHECK, and import_encoder.

1717  : col_reader_(col_reader)
1718  , column_descriptor_(column_descriptor)
1719  , parquet_column_descriptor_(parquet_column_descriptor)
1720  , encoder_(encoder)
1721  , invalid_indices_(invalid_indices)
1722  , row_group_index_(row_group_index)
1723  , parquet_column_index_(parquet_column_index)
1724  , parquet_reader_(parquet_reader) {
1725  import_encoder = dynamic_cast<ParquetImportEncoder*>(encoder);
1727  }
std::shared_ptr< parquet::ColumnReader > col_reader_
const parquet::ParquetFileReader * parquet_reader_
const parquet::ColumnDescriptor * parquet_column_descriptor_
#define CHECK(condition)
Definition: Logger.h:209

Member Function Documentation

void foreign_storage::ParquetRowGroupReader::eraseInvalidRowGroupData ( )
inline

Definition at line 1763 of file LazyParquetChunkLoader.cpp.

References foreign_storage::ParquetImportEncoder::eraseInvalidIndicesInBuffer(), import_encoder, and invalid_indices_.

1763  {
1765  }
virtual void eraseInvalidIndicesInBuffer(const InvalidRowGroupIndices &invalid_indices)=0

+ Here is the call graph for this function:

void foreign_storage::ParquetRowGroupReader::readAndValidateRowGroup ( )
inline

Definition at line 1729 of file LazyParquetChunkLoader.cpp.

References foreign_storage::LazyParquetChunkLoader::batch_reader_num_elements, col_reader_, column_descriptor_, ColumnDescriptor::columnType, foreign_storage::ParquetBatchData::def_levels, encoder_, SQLTypeInfo::get_subtype(), import_encoder, invalid_indices_, SQLTypeInfo::is_array(), foreign_storage::ParquetBatchData::levels_read, parquet_column_descriptor_, parquet_column_index_, parquet_reader_, foreign_storage::ParquetBatchData::rep_levels, foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::resize_values_buffer(), row_group_index_, foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_definition_levels(), foreign_storage::ParquetImportEncoder::validateAndAppendData(), foreign_storage::ParquetBatchData::values, and foreign_storage::ParquetBatchData::values_read.

1729  {
1730  while (col_reader_->HasNext()) {
1731  ParquetBatchData batch_data;
1733  column_descriptor_, parquet_column_descriptor_, batch_data.values);
1734  batch_data.levels_read =
1736  batch_data.def_levels.data(),
1737  batch_data.rep_levels.data(),
1738  reinterpret_cast<uint8_t*>(batch_data.values.data()),
1739  &batch_data.values_read,
1740  col_reader_.get());
1747  batch_data.def_levels.data(),
1748  batch_data.levels_read,
1750  import_encoder->validateAndAppendData(batch_data.def_levels.data(),
1751  batch_data.rep_levels.data(),
1752  batch_data.values_read,
1753  batch_data.levels_read,
1754  batch_data.values.data(),
1755  column_type,
1757  }
1758  if (auto array_encoder = dynamic_cast<ParquetArrayEncoder*>(encoder_)) {
1759  array_encoder->finalizeRowGroup();
1760  }
1761  }
std::shared_ptr< parquet::ColumnReader > col_reader_
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
const parquet::ParquetFileReader * parquet_reader_
const parquet::ColumnDescriptor * parquet_column_descriptor_
virtual void validateAndAppendData(const int16_t *def_levels, const int16_t *rep_levels, const int64_t values_read, const int64_t levels_read, int8_t *values, const SQLTypeInfo &column_type, InvalidRowGroupIndices &invalid_indices)=0
void resize_values_buffer(const ColumnDescriptor *omnisci_column, const parquet::ColumnDescriptor *parquet_column, std::vector< int8_t > &values)
SQLTypeInfo columnType
bool is_array() const
Definition: sqltypes.h:517
void validate_definition_levels(const parquet::ParquetFileReader *reader, const int row_group_index, const int column_index, const int16_t *def_levels, const int64_t num_levels, const parquet::ColumnDescriptor *parquet_column_descriptor)

+ Here is the call graph for this function:

Member Data Documentation

std::shared_ptr<parquet::ColumnReader> foreign_storage::ParquetRowGroupReader::col_reader_
private

Definition at line 1768 of file LazyParquetChunkLoader.cpp.

Referenced by readAndValidateRowGroup().

const ColumnDescriptor* foreign_storage::ParquetRowGroupReader::column_descriptor_
private

Definition at line 1769 of file LazyParquetChunkLoader.cpp.

Referenced by readAndValidateRowGroup().

ParquetEncoder* foreign_storage::ParquetRowGroupReader::encoder_
private

Definition at line 1771 of file LazyParquetChunkLoader.cpp.

Referenced by readAndValidateRowGroup().

ParquetImportEncoder* foreign_storage::ParquetRowGroupReader::import_encoder
private
InvalidRowGroupIndices& foreign_storage::ParquetRowGroupReader::invalid_indices_
private
const parquet::ColumnDescriptor* foreign_storage::ParquetRowGroupReader::parquet_column_descriptor_
private

Definition at line 1770 of file LazyParquetChunkLoader.cpp.

Referenced by readAndValidateRowGroup().

const int foreign_storage::ParquetRowGroupReader::parquet_column_index_
private

Definition at line 1775 of file LazyParquetChunkLoader.cpp.

Referenced by readAndValidateRowGroup().

const parquet::ParquetFileReader* foreign_storage::ParquetRowGroupReader::parquet_reader_
private

Definition at line 1776 of file LazyParquetChunkLoader.cpp.

Referenced by readAndValidateRowGroup().

const int foreign_storage::ParquetRowGroupReader::row_group_index_
private

Definition at line 1774 of file LazyParquetChunkLoader.cpp.

Referenced by readAndValidateRowGroup().


The documentation for this class was generated from the following file: