OmniSciDB  8a228a1076
LazyParquetImporter.cpp File Reference
#include "LazyParquetImporter.h"
#include <arrow/api.h>
#include <arrow/io/api.h>
#include <arrow/result.h>
#include <parquet/types.h>
#include "ImportExport/ArrowImporter.h"
#include "ParquetTypeMappings.h"
#include "Shared/ArrowUtil.h"
#include "Shared/measure.h"
#include "Shared/thread_count.h"
+ Include dependency graph for LazyParquetImporter.cpp:

Go to the source code of this file.

Namespaces

 foreign_storage
 
 foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}
 

Typedefs

using TypedImportBuffer = import_export::TypedImportBuffer
 

Functions

std::string foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::type_to_string (SQLTypes type)
 
std::tuple< int, int > foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::open_parquet_table (const std::string &file_path, std::unique_ptr< parquet::arrow::FileReader > &reader)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::validate_allowed_mapping (const parquet::ColumnDescriptor *descr, const ColumnDescriptor *cd)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::validate_parquet_metadata (const std::shared_ptr< parquet::FileMetaData > &file_metadata, const std::string &file_path, const ForeignTableSchema &schema)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::initialize_bad_rows_tracker (import_export::BadRowsTracker &bad_rows_tracker, const int row_group, const std::string &file_path, import_export::Importer *importer)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::initialize_import_buffers_vec (const import_export::Loader *loader, const ForeignTableSchema &schema, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffer_vec)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::initialize_row_group_metadata_vec (const size_t num_logical_and_physical_columns, LazyParquetImporter::RowGroupMetadataVector &row_group_metadata_vec)
 
std::unique_ptr< import_export::TypedImportBuffer > & foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::initialize_import_buffer (const size_t column_id, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffer_vec)
 
int64_t foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::get_divisor (const SQLTypeInfo &type, const parquet::LogicalType::TimeUnit::unit time_unit)
 
std::pair< int64_t, int64_t > foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::get_datetime_min_and_max (const ColumnDescriptor *column_descriptor, const parquet::ColumnDescriptor *parquet_column_descriptor, std::shared_ptr< parquet::Statistics > stats)
 
int64_t foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::convert_decimal_byte_array_to_int (const std::string &encoded_value)
 
std::pair< int64_t, int64_t > foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::get_decimal_min_and_max (const ColumnDescriptor *column_descriptor, const parquet::ColumnDescriptor *parquet_column_descriptor, std::shared_ptr< parquet::Statistics > stats)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::read_parquet_metadata_into_import_buffer (const size_t num_rows, const int row_group, const ForeignTableSchema &schema, const std::unique_ptr< parquet::RowGroupMetaData > &group_metadata, const ColumnDescriptor *column_descriptor, import_export::BadRowsTracker &bad_rows_tracker, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffer_vec, LazyParquetImporter::RowGroupMetadataVector &row_group_metadata_vec)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::read_parquet_data_into_import_buffer (const import_export::Loader *loader, const int row_group, const ForeignTableSchema &schema, const ColumnDescriptor *column_descriptor, std::unique_ptr< parquet::arrow::FileReader > &reader, import_export::BadRowsTracker &bad_rows_tracker, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffer_vec)
 
void foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::import_row_group (const int row_group, const bool metadata_scan, const ForeignTableSchema &schema, const std::string &file_path, const Interval< ColumnType > &column_interval, import_export::Importer *importer, std::unique_ptr< parquet::arrow::FileReader > &reader, LazyParquetImporter::RowGroupMetadataVector &row_group_metadata_vec, import_export::BadRowsTracker &bad_rows_tracker, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffer_vec)
 

Variables

constexpr int foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::MILLIS_PER_SECOND = 1000
 
constexpr int foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::MICROS_PER_SECOND = 1000 * 1000
 
constexpr int foreign_storage::anonymous_namespace{LazyParquetImporter.cpp}::NANOS_PER_SECOND = 1000 * 1000 * 1000
 

Typedef Documentation

◆ TypedImportBuffer