#include <LazyParquetChunkLoader.h>

Public Member Functions
	LazyParquetChunkLoader (std::shared_ptr< arrow::fs::FileSystem > file_system)

std::list< std::unique_ptr < ChunkMetadata > >	loadChunk (const std::vector< RowGroupInterval > &row_group_intervals, const int parquet_column_index, std::list< Chunk_NS::Chunk > &chunks, StringDictionary *string_dictionary=nullptr)

std::list< RowGroupMetadata >	metadataScan (const std::set< std::string > &file_paths, const ForeignTableSchema &schema)
	Perform a metadata scan for the paths specified. More...

Static Public Member Functions
static bool	isColumnMappingSupported (const ColumnDescriptor omnisci_column, const parquet::ColumnDescriptor parquet_column)

Static Public Attributes
static const int	batch_reader_num_elements = 4096

Private Attributes
std::shared_ptr < arrow::fs::FileSystem >	file_system_

Detailed Description

A lazy parquet to chunk loader

Definition at line 32 of file LazyParquetChunkLoader.h.

Constructor & Destructor Documentation

foreign_storage::LazyParquetChunkLoader::LazyParquetChunkLoader ( std::shared_ptr< arrow::fs::FileSystem > file_system )

Definition at line 1478 of file LazyParquetChunkLoader.cpp.

1480 : file_system_(file_system) {}

foreign_storage::LazyParquetChunkLoader::file_system_

std::shared_ptr< arrow::fs::FileSystem > file_system_

Definition: LazyParquetChunkLoader.h:94

Member Function Documentation

bool foreign_storage::LazyParquetChunkLoader::isColumnMappingSupported	(	const ColumnDescriptor *	omnisci_column,
		const parquet::ColumnDescriptor *	parquet_column
	)

static

Determine if a Parquet to OmniSci column mapping is supported.

Parameters

omnisci_column	- the column descriptor of the OmniSci column
parquet_column	- the column descriptor of the Parquet column

Returns: true if the column mapping is supported by LazyParquetChunkLoader, false otherwise

Definition at line 1442 of file LazyParquetChunkLoader.cpp.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_allowed_mapping(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_array_mapping().

                                                    {
   if (validate_geospatial_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_array_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_decimal_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_floating_point_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_integral_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_none_type_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_timestamp_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_time_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_date_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   if (validate_string_mapping(omnisci_column, parquet_column)) {
     return true;
   }
   return false;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::list< std::unique_ptr< ChunkMetadata > > foreign_storage::LazyParquetChunkLoader::loadChunk	(	const std::vector< RowGroupInterval > &	row_group_intervals,
		const int	parquet_column_index,
		std::list< Chunk_NS::Chunk > &	chunks,
		StringDictionary *	string_dictionary = `nullptr`
	)

Load a number of row groups of a column in a parquet file into a chunk

Parameters

row_group_interval	- an inclusive interval [start,end] that specifies row groups to load
parquet_column_index	- the logical column index in the parquet file (and omnisci db) of column to load
chunks	- a list containing the chunks to load
string_dictionary	- a string dictionary for the column corresponding to the column, if applicable

Returns: An empty list when no metadata update is applicable, otherwise a list of ChunkMetadata shared pointers with which to update the corresponding column chunk metadata. NOTE: Only ChunkMetadata.sqlType and the min & max values of the ChunkMetadata.chunkStats are valid, other values are not set.

NOTE: if more than one chunk is supplied, the first chunk is required to be the chunk corresponding to the logical column, while the remaining chunks correspond to physical columns (in ascending order of column id.) Similarly, if a metada update is expected, the list of ChunkMetadata shared pointers returned will correspond directly to the list chunks.

Definition at line 1482 of file LazyParquetChunkLoader.cpp.

References foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::append_row_groups(), CHECK, and file_system_.

Referenced by foreign_storage::ParquetDataWrapper::loadBuffersUsingLazyParquetChunkLoader().

                                          {
   CHECK(!chunks.empty());
   auto const& chunk = *chunks.begin();
   auto column_descriptor = chunk.getColumnDesc();
   auto buffer = chunk.getBuffer();
   CHECK(buffer);
 
   try {
     auto metadata = append_row_groups(row_group_intervals,
                                       parquet_column_index,
                                       column_descriptor,
                                       chunks,
                                       string_dictionary,
                                       file_system_);
     return metadata;
   } catch (const std::exception& error) {
     throw ForeignStorageException(error.what());
   }
 
   return {};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::list< RowGroupMetadata > foreign_storage::LazyParquetChunkLoader::metadataScan	(	const std::set< std::string > &	file_paths,
		const ForeignTableSchema &	schema
	)

Perform a metadata scan for the paths specified.

Parameters

file_paths	- (ordered) files of the metadata scan
schema	- schema of the foreign table to perform metadata scan for

Returns: a list of the row group metadata extracted from file_paths

Definition at line 1508 of file LazyParquetChunkLoader.cpp.

References CHECK, file_system_, foreign_storage::get_parquet_table_size(), foreign_storage::ForeignTableSchema::getLogicalAndPhysicalColumns(), foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::metadata_scan_rowgroup_interval(), foreign_storage::open_parquet_table(), foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::populate_encoder_map(), foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_parquet_metadata().

Referenced by foreign_storage::ParquetDataWrapper::metadataScanFiles().

                                       {
   std::list<RowGroupMetadata> row_group_metadata;
   auto column_interval =
       Interval<ColumnType>{schema.getLogicalAndPhysicalColumns().front()->columnId,
                            schema.getLogicalAndPhysicalColumns().back()->columnId};
   CHECK(!file_paths.empty());
   std::unique_ptr<parquet::arrow::FileReader> first_file_reader;
   const auto& first_file_path = *file_paths.begin();
   open_parquet_table(first_file_path, first_file_reader, file_system_);
   std::map<int, std::shared_ptr<ParquetEncoder>> encoder_map;
   for (const auto& file_path : file_paths) {
     std::unique_ptr<parquet::arrow::FileReader> reader;
     open_parquet_table(file_path, reader, file_system_);
     validate_equal_schema(
         first_file_reader.get(), reader.get(), first_file_path, file_path);
     int num_row_groups = get_parquet_table_size(reader).first;
     auto row_group_interval = RowGroupInterval{file_path, 0, num_row_groups - 1};
     validate_parquet_metadata(reader->parquet_reader()->metadata(), file_path, schema);
     if (file_path == first_file_path) {
       populate_encoder_map(encoder_map, column_interval, schema, first_file_reader);
     }
     metadata_scan_rowgroup_interval(
         encoder_map, row_group_interval, reader, schema, row_group_metadata);
   }
   return row_group_metadata;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

const int foreign_storage::LazyParquetChunkLoader::batch_reader_num_elements = 4096

static

Definition at line 37 of file LazyParquetChunkLoader.h.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::append_row_groups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::resize_values_buffer().

std::shared_ptr<arrow::fs::FileSystem> foreign_storage::LazyParquetChunkLoader::file_system_

private

Definition at line 94 of file LazyParquetChunkLoader.h.

Referenced by loadChunk(), and metadataScan().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/LazyParquetChunkLoader.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/LazyParquetChunkLoader.cpp

Public Member Functions

Static Public Member Functions

Static Public Attributes

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation