Inheritance diagram for foreign_storage::RowGroupIntervalTracker:

Collaboration diagram for foreign_storage::RowGroupIntervalTracker:

Public Member Functions
	RowGroupIntervalTracker (const std::set< std::string > &file_paths, FileReaderMap *file_reader_cache, std::shared_ptr< arrow::fs::FileSystem > file_system)

std::optional< RowGroupInterval >	getNextRowGroupInterval () override

Public Member Functions inherited from foreign_storage::AbstractRowGroupIntervalTracker
virtual	~AbstractRowGroupIntervalTracker ()=default

Private Member Functions
bool	filesAreExhausted ()

void	advanceToNextRowGroup ()

Private Attributes
std::set< std::string >	file_paths_

FileReaderMap *	file_reader_cache_

std::shared_ptr < arrow::fs::FileSystem >	file_system_

bool	is_initialized_

int	num_row_groups_

int	current_row_group_index_

std::set< std::string > ::const_iterator	current_file_iter_

Detailed Description

Definition at line 37 of file ParquetImporter.cpp.

Constructor & Destructor Documentation

foreign_storage::RowGroupIntervalTracker::RowGroupIntervalTracker	(	const std::set< std::string > &	file_paths,
		FileReaderMap *	file_reader_cache,
		std::shared_ptr< arrow::fs::FileSystem >	file_system
	)

inline

Definition at line 39 of file ParquetImporter.cpp.

       : file_paths_(file_paths)
       , file_reader_cache_(file_reader_cache)
       , file_system_(file_system)
       , is_initialized_(false)
       , num_row_groups_(0)
       , current_row_group_index_(0)
       , current_file_iter_(file_paths_.begin()) {}

Member Function Documentation

void foreign_storage::RowGroupIntervalTracker::advanceToNextRowGroup ( )

inlineprivate

Definition at line 62 of file ParquetImporter.cpp.

References current_file_iter_, current_row_group_index_, file_paths_, file_reader_cache_, file_system_, filesAreExhausted(), foreign_storage::FileReaderMap::getOrInsert(), is_initialized_, and num_row_groups_.

Referenced by getNextRowGroupInterval().

                                {
     if (current_row_group_index_ < num_row_groups_ - 1) {
       current_row_group_index_++;
       return;
     }
     if (!is_initialized_) {
       current_file_iter_ = file_paths_.begin();
       is_initialized_ = true;
     } else {
       if (filesAreExhausted()) {  // can be possible if many concurrent requests
         return;
       }
       current_file_iter_++;  // advance iterator
     }
     current_row_group_index_ = 0;
     if (filesAreExhausted()) {
       num_row_groups_ = 0;
     } else {
       auto file_reader =
           file_reader_cache_->getOrInsert(*current_file_iter_, file_system_);
       num_row_groups_ = file_reader->parquet_reader()->metadata()->num_row_groups();
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::RowGroupIntervalTracker::filesAreExhausted ( )

inlineprivate

Definition at line 60 of file ParquetImporter.cpp.

References current_file_iter_, and file_paths_.

Referenced by advanceToNextRowGroup(), and getNextRowGroupInterval().

60 { return current_file_iter_ == file_paths_.end(); }

foreign_storage::RowGroupIntervalTracker::file_paths_

std::set< std::string > file_paths_

Definition: ParquetImporter.cpp:86

foreign_storage::RowGroupIntervalTracker::current_file_iter_

std::set< std::string >::const_iterator current_file_iter_

Definition: ParquetImporter.cpp:93

Here is the caller graph for this function:

std::optional<RowGroupInterval> foreign_storage::RowGroupIntervalTracker::getNextRowGroupInterval ( )

inlineoverridevirtual

Implements foreign_storage::AbstractRowGroupIntervalTracker.

Definition at line 50 of file ParquetImporter.cpp.

References advanceToNextRowGroup(), current_file_iter_, current_row_group_index_, and filesAreExhausted().

                                                                    {
     advanceToNextRowGroup();
     if (filesAreExhausted()) {
       return {};
     }
     return RowGroupInterval{
         *current_file_iter_, current_row_group_index_, current_row_group_index_};
   }