#include <FileReader.h>

Inheritance diagram for foreign_storage::MultiFileReader:

Collaboration diagram for foreign_storage::MultiFileReader:

Public Member Functions
	MultiFileReader (const std::string &file_path, const import_export::CopyParams &copy_params)

	MultiFileReader (const std::string &file_path, const import_export::CopyParams &copy_params, const rapidjson::Value &value)

size_t	getRemainingSize () override

bool	isRemainingSizeKnown () override

size_t	read (void *buffer, size_t max_size) override

size_t	readRegion (void *buffer, size_t offset, size_t size) override

bool	isScanFinished () const override

void	serialize (rapidjson::Value &value, rapidjson::Document::AllocatorType &allocator) const override

FirstLineByFilePath	getFirstLineForEachFile () const override

bool	isEndOfLastFile () override

std::string	getCurrentFilePath () const override

virtual std::set< std::string >	checkForRolledOffFiles (const shared::FilePathOptions &file_path_options)

Public Member Functions inherited from foreign_storage::FileReader
	FileReader (const std::string &file_path, const import_export::CopyParams &copy_params)

virtual	~FileReader ()=default

virtual void	checkForMoreRows (size_t file_offset, const shared::FilePathOptions &options, const ForeignServer server_options=nullptr, const UserMapping user_mapping=nullptr)

Protected Member Functions
virtual std::vector< std::string >	getAllFilePaths (const shared::FilePathOptions &file_path_options) const =0

Protected Attributes
std::vector< std::unique_ptr < FileReader > >	files_

std::vector< std::string >	file_locations_

std::vector< size_t >	cumulative_sizes_

size_t	current_index_

size_t	current_offset_

size_t	starting_offset_

bool	is_end_of_last_file_

Protected Attributes inherited from foreign_storage::FileReader
import_export::CopyParams	copy_params_

std::string	file_path_

Detailed Description

Definition at line 346 of file FileReader.h.

Constructor & Destructor Documentation

foreign_storage::MultiFileReader::MultiFileReader	(	const std::string &	file_path,
		const import_export::CopyParams &	copy_params
	)

Definition at line 530 of file FileReader.cpp.

     : FileReader(file_path, copy_params)
     , current_index_(0)
     , current_offset_(0)
     , starting_offset_(0)
     , is_end_of_last_file_(false) {}

foreign_storage::MultiFileReader::MultiFileReader	(	const std::string &	file_path,
		const import_export::CopyParams &	copy_params,
		const rapidjson::Value &	value
	)

Definition at line 538 of file FileReader.cpp.

References CHECK, cumulative_sizes_, current_index_, current_offset_, file_locations_, json_utils::get_value_from_object(), and starting_offset_.

     : FileReader(file_path, copy_params)
     , current_index_(0)
     , current_offset_(0)
     , starting_offset_(0)
     , is_end_of_last_file_(false) {
   json_utils::get_value_from_object(value, file_locations_, "file_locations");
   json_utils::get_value_from_object(value, cumulative_sizes_, "cumulative_sizes");
   json_utils::get_value_from_object(value, current_offset_, "current_offset");
   json_utils::get_value_from_object(value, current_index_, "current_index");
   if (value.HasMember("starting_offset")) {
     json_utils::get_value_from_object(value, starting_offset_, "starting_offset");
   }
 
   // Validate files_metadata here, but objects will be recreated by child class
   CHECK(value.HasMember("files_metadata"));
   CHECK(value["files_metadata"].IsArray());
   CHECK(file_locations_.size() == value["files_metadata"].GetArray().Size());
 }

Here is the call graph for this function:

Member Function Documentation

std::set< std::string > foreign_storage::MultiFileReader::checkForRolledOffFiles ( const shared::FilePathOptions & file_path_options )

virtual

Definition at line 615 of file FileReader.cpp.

References shared::check_for_rolled_off_file_paths(), CHECK_LE, cumulative_sizes_, current_index_, file_locations_, files_, getAllFilePaths(), and starting_offset_.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                                     {
   auto all_file_paths = getAllFilePaths(file_path_options);
   auto rolled_off_files =
       shared::check_for_rolled_off_file_paths(all_file_paths, file_locations_);
   if (!rolled_off_files.empty()) {
     files_.erase(files_.begin(), files_.begin() + rolled_off_files.size());
     CHECK_LE(rolled_off_files.size(), cumulative_sizes_.size());
     starting_offset_ = cumulative_sizes_[rolled_off_files.size() - 1];
     cumulative_sizes_.erase(cumulative_sizes_.begin(),
                             cumulative_sizes_.begin() + rolled_off_files.size());
     current_index_ -= rolled_off_files.size();
   }
   return rolled_off_files;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual std::vector<std::string> foreign_storage::MultiFileReader::getAllFilePaths ( const shared::FilePathOptions & file_path_options ) const

protectedpure virtual

Implemented in foreign_storage::LocalMultiFileReader.

Referenced by checkForRolledOffFiles().

Here is the caller graph for this function:

std::string foreign_storage::MultiFileReader::getCurrentFilePath ( ) const

overridevirtual

Returns the path of the currently processed file.

Implements foreign_storage::FileReader.

Definition at line 607 of file FileReader.cpp.

References CHECK_LT, current_index_, files_, and isScanFinished().

                                                     {
   if (isScanFinished()) {
     return files_.back()->getCurrentFilePath();
   }
   CHECK_LT(current_index_, files_.size());
   return files_[current_index_]->getCurrentFilePath();
 }

Here is the call graph for this function:

FirstLineByFilePath foreign_storage::MultiFileReader::getFirstLineForEachFile ( ) const

overridevirtual

Returns a map containing the first line for each file that will be read.

Implements foreign_storage::FileReader.

Definition at line 595 of file FileReader.cpp.

References files_.

                                                                    {
   FirstLineByFilePath first_line_by_file_path;
   for (const auto& file : files_) {
     first_line_by_file_path.merge(file->getFirstLineForEachFile());
   }
   return first_line_by_file_path;
 }

size_t foreign_storage::MultiFileReader::getRemainingSize ( )

overridevirtual

Returns: size of the remaining content to be read

Implements foreign_storage::FileReader.

Definition at line 579 of file FileReader.cpp.

References current_index_, and files_.

                                          {
   size_t total_size = 0;
   for (size_t index = current_index_; index < files_.size(); index++) {
     total_size += files_[index]->getRemainingSize();
   }
   return total_size;
 }

bool foreign_storage::MultiFileReader::isEndOfLastFile ( )

overridevirtual

Returns a boolean indicating whether the reader is at the end of the last file that was read.

Implements foreign_storage::FileReader.

Definition at line 603 of file FileReader.cpp.

References is_end_of_last_file_, and isScanFinished().

                                       {
   return (isScanFinished() || is_end_of_last_file_);
 }

Here is the call graph for this function:

bool foreign_storage::MultiFileReader::isRemainingSizeKnown ( )

overridevirtual

Returns: if remaining size is known

Implements foreign_storage::FileReader.

Definition at line 587 of file FileReader.cpp.

References current_index_, and files_.

                                            {
   bool size_known = true;
   for (size_t index = current_index_; index < files_.size(); index++) {
     size_known = size_known && files_[index]->isRemainingSizeKnown();
   }
   return size_known;
 }

bool foreign_storage::MultiFileReader::isScanFinished ( ) const

inlineoverridevirtual

Returns: true if the entire file has been read

Implements foreign_storage::FileReader.

Definition at line 362 of file FileReader.h.

References current_index_, and files_.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), getCurrentFilePath(), isEndOfLastFile(), read(), and readRegion().

362 { return (current_index_ >= files_.size()); }

foreign_storage::MultiFileReader::current_index_

size_t current_index_

Definition: FileReader.h:386

foreign_storage::MultiFileReader::files_

std::vector< std::unique_ptr< FileReader > > files_

Definition: FileReader.h:380

Here is the caller graph for this function:

size_t foreign_storage::MultiFileReader::read	(	void *	buffer,
		size_t	max_size
	)

overridevirtual

Read up to max_size bytes from archive into buffer starting starting from the end of the last read

Parameters

buffer	- buffer to load into
max_size	- maximum number of bytes to read into the buffer

Returns: number of bytes actually read

Implements foreign_storage::FileReader.

Definition at line 734 of file FileReader.cpp.

References foreign_storage::anonymous_namespace{FileReader.cpp}::adjust_eof(), foreign_storage::FileReader::copy_params_, cumulative_sizes_, current_index_, current_offset_, files_, is_end_of_last_file_, isScanFinished(), and import_export::CopyParams::line_delim.

                                                           {
   if (isScanFinished()) {
     return 0;
   }
   // Leave one extra char in case we need to insert a delimiter
   size_t bytes_read = files_[current_index_].get()->read(buffer, max_size - 1);
   if (files_[current_index_].get()->isScanFinished()) {
     adjust_eof(bytes_read, max_size, static_cast<char*>(buffer), copy_params_.line_delim);
   }
   current_offset_ += bytes_read;
   if (current_index_ < files_.size() && files_[current_index_].get()->isScanFinished()) {
     cumulative_sizes_.push_back(current_offset_);
     current_index_++;
     is_end_of_last_file_ = true;
   } else {
     is_end_of_last_file_ = false;
   }
   return bytes_read;
 }

Here is the call graph for this function:

size_t foreign_storage::MultiFileReader::readRegion	(	void *	buffer,
		size_t	offset,
		size_t	size
	)

overridevirtual

Read up to max_size bytes from archive, starting at given offset isScanFinished() must return true to use readRegion

Parameters

buffer	- buffer to load into
offset	- starting point into the archive to read
size	- maximum number of bytes to read into the buffer

Returns: number of bytes actually read

Implements foreign_storage::FileReader.

Definition at line 754 of file FileReader.cpp.

References CHECK, foreign_storage::FileReader::copy_params_, cumulative_sizes_, files_, isScanFinished(), import_export::CopyParams::line_delim, foreign_storage::anonymous_namespace{FileReader.cpp}::offset_to_index(), and starting_offset_.

                                                                            {
   CHECK(isScanFinished());
   // Get file index
   auto index = offset_to_index(cumulative_sizes_, offset);
   // Get offset into this file
   size_t base = starting_offset_;
   if (index > 0) {
     base = cumulative_sizes_[index - 1];
   }
 
   size_t read_size = size;
   if (offset + size == cumulative_sizes_[index]) {
     // Skip the last byte as it may have been an inserted delimiter
     read_size--;
   }
   size_t bytes_read = files_[index].get()->readRegion(buffer, offset - base, read_size);
 
   if (offset + size == cumulative_sizes_[index]) {
     // Re-insert delimiter
     static_cast<char*>(buffer)[size - 1] = copy_params_.line_delim;
     bytes_read++;
   }
 
   return bytes_read;
 }

Here is the call graph for this function:

void foreign_storage::MultiFileReader::serialize	(	rapidjson::Value &	value,
		rapidjson::Document::AllocatorType &	allocator
	)		const

overridevirtual

Serialize internal state to given json object This Json will later be used to restore the reader state through a constructor must be called when isScanFinished() is true

Parameters

value	- json object to store needed state to this function can store any needed data or none
allocator	- allocator to use for json contruction

Implements foreign_storage::FileReader.

Definition at line 560 of file FileReader.cpp.

References json_utils::add_value_to_object(), cumulative_sizes_, current_index_, current_offset_, file_locations_, files_, and starting_offset_.

                                                                                  {
   json_utils::add_value_to_object(value, file_locations_, "file_locations", allocator);
   json_utils::add_value_to_object(
       value, cumulative_sizes_, "cumulative_sizes", allocator);
   json_utils::add_value_to_object(value, current_offset_, "current_offset", allocator);
   json_utils::add_value_to_object(value, current_index_, "current_index", allocator);
   json_utils::add_value_to_object(value, starting_offset_, "starting_offset", allocator);
 
   // Serialize metadata from all files
   rapidjson::Value files_metadata(rapidjson::kArrayType);
   for (size_t index = 0; index < files_.size(); index++) {
     rapidjson::Value file_metadata(rapidjson::kObjectType);
     files_[index]->serialize(file_metadata, allocator);
     files_metadata.PushBack(file_metadata, allocator);
   }
   value.AddMember("files_metadata", files_metadata, allocator);
 };

Here is the call graph for this function:

Member Data Documentation

std::vector<size_t> foreign_storage::MultiFileReader::cumulative_sizes_

protected

Definition at line 384 of file FileReader.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), checkForRolledOffFiles(), MultiFileReader(), read(), readRegion(), and serialize().

size_t foreign_storage::MultiFileReader::current_index_

protected

Definition at line 386 of file FileReader.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), checkForRolledOffFiles(), getCurrentFilePath(), getRemainingSize(), isRemainingSizeKnown(), isScanFinished(), MultiFileReader(), read(), and serialize().

size_t foreign_storage::MultiFileReader::current_offset_

protected

Definition at line 388 of file FileReader.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), MultiFileReader(), read(), and serialize().

std::vector<std::string> foreign_storage::MultiFileReader::file_locations_

protected

Definition at line 381 of file FileReader.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), checkForRolledOffFiles(), foreign_storage::LocalMultiFileReader::insertFile(), foreign_storage::LocalMultiFileReader::LocalMultiFileReader(), MultiFileReader(), and serialize().

std::vector<std::unique_ptr<FileReader> > foreign_storage::MultiFileReader::files_

protected

Definition at line 380 of file FileReader.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), checkForRolledOffFiles(), getCurrentFilePath(), getFirstLineForEachFile(), getRemainingSize(), foreign_storage::LocalMultiFileReader::insertFile(), isRemainingSizeKnown(), isScanFinished(), foreign_storage::LocalMultiFileReader::LocalMultiFileReader(), read(), readRegion(), and serialize().

bool foreign_storage::MultiFileReader::is_end_of_last_file_

protected

Definition at line 392 of file FileReader.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), isEndOfLastFile(), and read().

size_t foreign_storage::MultiFileReader::starting_offset_

protected

Definition at line 390 of file FileReader.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), checkForRolledOffFiles(), MultiFileReader(), readRegion(), and serialize().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/FileReader.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/FileReader.cpp

Public Member Functions

Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation