#include <CsvReader.h>

Inheritance diagram for foreign_storage::SingleFileReader:

Collaboration diagram for foreign_storage::SingleFileReader:

Public Member Functions
	SingleFileReader (const std::string &file_path, const import_export::CopyParams &copy_params)

	SingleFileReader (const std::string &file_path, const import_export::CopyParams &copy_params, const rapidjson::Value &value)

	~SingleFileReader () override

	SingleFileReader (const SingleFileReader &)=delete

SingleFileReader &	operator= (const SingleFileReader &)=delete

size_t	read (void *buffer, size_t max_size) override

size_t	readRegion (void *buffer, size_t offset, size_t size) override

bool	isScanFinished () override

size_t	getRemainingSize () override

bool	isRemainingSizeKnown () override

void	checkForMoreRows (size_t file_offset, const ForeignServer server_options, const UserMapping user_mapping) override

void	serialize (rapidjson::Value &value, rapidjson::Document::AllocatorType &allocator) const override

Public Member Functions inherited from foreign_storage::CsvReader
	CsvReader (const std::string &file_path, const import_export::CopyParams &copy_params)

virtual	~CsvReader ()=default

Private Attributes
std::FILE *	file_

size_t	data_size_

bool	scan_finished_

size_t	header_offset_

size_t	total_bytes_read_

Additional Inherited Members
Protected Attributes inherited from foreign_storage::CsvReader
import_export::CopyParams	copy_params_

std::string	file_path_

Detailed Description

Definition at line 107 of file CsvReader.h.

Constructor & Destructor Documentation

foreign_storage::SingleFileReader::SingleFileReader	(	const std::string &	file_path,
		const import_export::CopyParams &	copy_params
	)

Definition at line 69 of file CsvReader.cpp.

References CHECK, data_size_, file_, foreign_storage::anonymous_namespace{CsvReader.cpp}::get_data_size(), import_export::CopyParams::has_header, header_offset_, parse_ast::line, import_export::CopyParams::line_delim, and import_export::NO_HEADER.

     : CsvReader(file_path, copy_params)
     , scan_finished_(false)
     , header_offset_(0)
     , total_bytes_read_(0) {
   file_ = fopen(file_path.c_str(), "rb");
   if (!file_) {
     throw std::runtime_error{"An error occurred when attempting to open file \"" +
                              file_path + "\". " + strerror(errno)};
   }
 
   // Skip header and record offset
   if (copy_params.has_header != import_export::ImportHeaderRow::NO_HEADER) {
     std::ifstream file{file_path};
     CHECK(file.good());
     std::string line;
     std::getline(file, line, copy_params.line_delim);
     file.close();
     header_offset_ = line.size() + 1;
   }
   fseek(file_, 0, SEEK_END);
 
   data_size_ = get_data_size(ftell(file_), header_offset_);
 
   if (fseek(file_, static_cast<long int>(header_offset_), SEEK_SET) != 0) {
     throw std::runtime_error{"An error occurred when attempting to open file \"" +
                              file_path + "\". " + strerror(errno)};
   };
 }

Here is the call graph for this function:

foreign_storage::SingleFileReader::SingleFileReader	(	const std::string &	file_path,
		const import_export::CopyParams &	copy_params,
		const rapidjson::Value &	value
	)

Definition at line 100 of file CsvReader.cpp.

References data_size_, file_, foreign_storage::json_utils::get_value_from_object(), header_offset_, and total_bytes_read_.

     : CsvReader(file_path, copy_params)
     , scan_finished_(true)
     , header_offset_(0)
     , total_bytes_read_(0) {
   file_ = fopen(file_path.c_str(), "rb");
   if (!file_) {
     throw std::runtime_error{"An error occurred when attempting to open file \"" +
                              file_path + "\". " + strerror(errno)};
   }
   json_utils::get_value_from_object(value, header_offset_, "header_offset");
   json_utils::get_value_from_object(value, total_bytes_read_, "total_bytes_read");
   json_utils::get_value_from_object(value, data_size_, "data_size");
 }

Here is the call graph for this function:

foreign_storage::SingleFileReader::~SingleFileReader ( )

inlineoverride

Definition at line 114 of file CsvReader.h.

References file_.

114 { fclose(file_); }

foreign_storage::SingleFileReader::file_

std::FILE * file_

Definition: CsvReader.h:153

foreign_storage::SingleFileReader::SingleFileReader ( const SingleFileReader & )

delete

Member Function Documentation

void foreign_storage::SingleFileReader::checkForMoreRows	(	size_t	file_offset,
		const ForeignServer *	server_options,
		const UserMapping *	user_mapping
	)

overridevirtual

Rescan the target files Throws an exception if the rescan fails (ie files are not in a valid appended state or not supported)

Parameters

file_offset	- where to resume the scan from (end of the last row) as not all of the bytes may have been consumed by the upstream compoennet
server_options	- only needed for S3 backed CSV
user_mapping	- only needed for S3 backed CSV

Reimplemented from foreign_storage::CsvReader.

Definition at line 126 of file CsvReader.cpp.

References CHECK, data_size_, file_, foreign_storage::CsvReader::file_path_, foreign_storage::anonymous_namespace{CsvReader.cpp}::get_data_size(), header_offset_, isScanFinished(), scan_finished_, foreign_storage::throw_removed_row_error(), to_string(), and total_bytes_read_.

                                                                          {
   CHECK(isScanFinished());
   // Re-open file and check if there is any new data in it
   fclose(file_);
   file_ = fopen(file_path_.c_str(), "rb");
   if (!file_) {
     throw std::runtime_error{"An error occurred when attempting to open file \"" +
                              file_path_ + "\". " + strerror(errno)};
   }
   fseek(file_, 0, SEEK_END);
   size_t new_file_size = ftell(file_);
   size_t new_data_size = get_data_size(new_file_size, header_offset_);
   if (new_data_size < data_size_) {
     throw_removed_row_error(file_path_);
   }
   if (fseek(file_, static_cast<long int>(file_offset + header_offset_), SEEK_SET) != 0) {
     throw std::runtime_error{"An error occurred when attempting to read offset " +
                              std::to_string(file_offset + header_offset_) +
                              " in file: \"" + file_path_ + "\". " + strerror(errno)};
   }
   if (new_data_size > data_size_) {
     scan_finished_ = false;
     total_bytes_read_ = file_offset;
     data_size_ = new_data_size;
   }
 }

Here is the call graph for this function:

size_t foreign_storage::SingleFileReader::getRemainingSize ( )

inlineoverridevirtual

Returns: size of the CSV remaining to be read

Implements foreign_storage::CsvReader.

Definition at line 142 of file CsvReader.h.

References data_size_, and total_bytes_read_.

142 { return data_size_ - total_bytes_read_; }

foreign_storage::SingleFileReader::total_bytes_read_

size_t total_bytes_read_

Definition: CsvReader.h:162

foreign_storage::SingleFileReader::data_size_

size_t data_size_

Definition: CsvReader.h:155

bool foreign_storage::SingleFileReader::isRemainingSizeKnown ( )

inlineoverridevirtual

Returns: if remaining size is known

Implements foreign_storage::CsvReader.

Definition at line 144 of file CsvReader.h.

144 { return true; };

bool foreign_storage::SingleFileReader::isScanFinished ( )

inlineoverridevirtual

Returns: true if the entire CSV has been read

Implements foreign_storage::CsvReader.

Definition at line 140 of file CsvReader.h.

References scan_finished_.

Referenced by checkForMoreRows(), and readRegion().

140 { return scan_finished_; }

foreign_storage::SingleFileReader::scan_finished_

bool scan_finished_

Definition: CsvReader.h:157

Here is the caller graph for this function:

SingleFileReader& foreign_storage::SingleFileReader::operator= ( const SingleFileReader & )

delete

size_t foreign_storage::SingleFileReader::read	(	void *	buffer,
		size_t	max_size
	)

inlineoverridevirtual

Read up to max_size bytes from archive into buffer starting starting from the end of the last read

Parameters

buffer	- buffer to load into
max_size	- maximum number of bytes to read into the buffer

Returns: number of bytes actually read

Implements foreign_storage::CsvReader.

Definition at line 120 of file CsvReader.h.

References file_, scan_finished_, and total_bytes_read_.

                                                       {
     size_t bytes_read = fread(buffer, 1, max_size, file_);
     if (!scan_finished_) {
       scan_finished_ = feof(file_);
     }
 
     total_bytes_read_ += bytes_read;
     return bytes_read;
   }

size_t foreign_storage::SingleFileReader::readRegion	(	void *	buffer,
		size_t	offset,
		size_t	size
	)

inlineoverridevirtual

Read up to max_size bytes from archive, starting at given offset isScanFinished() must return true to use readRegion

Parameters

buffer	- buffer to load into
offset	- starting point into the archive to read
size	- maximum number of bytes to read into the buffer

Returns: number of bytes actually read

Implements foreign_storage::CsvReader.

Definition at line 130 of file CsvReader.h.

References CHECK, file_, foreign_storage::CsvReader::file_path_, header_offset_, isScanFinished(), and to_string().

                                                                        {
     CHECK(isScanFinished());
     if (fseek(file_, static_cast<long int>(offset + header_offset_), SEEK_SET) != 0) {
       throw std::runtime_error{"An error occurred when attempting to read offset " +
                                std::to_string(offset) + " in file: \"" + file_path_ +
                                "\". " + strerror(errno)};
     }
     return fread(buffer, 1, size, file_);
   }

Here is the call graph for this function:

void foreign_storage::SingleFileReader::serialize	(	rapidjson::Value &	value,
		rapidjson::Document::AllocatorType &	allocator
	)		const

overridevirtual

Serialize internal state to given json object This Json will later be used to restore the reader state through a constructor must be called when isScanFinished() is true

Parameters

value	- json object to store needed state to this function can store any needed data or none
allocator	- allocator to use for json contruction

Implements foreign_storage::CsvReader.

Definition at line 117 of file CsvReader.cpp.

References foreign_storage::json_utils::add_value_to_object(), CHECK, data_size_, header_offset_, scan_finished_, and total_bytes_read_.

                                                                                   {
   CHECK(scan_finished_);
   json_utils::add_value_to_object(value, header_offset_, "header_offset", allocator);
   json_utils::add_value_to_object(
       value, total_bytes_read_, "total_bytes_read", allocator);
   json_utils::add_value_to_object(value, data_size_, "data_size", allocator);
 };

Here is the call graph for this function:

Member Data Documentation

size_t foreign_storage::SingleFileReader::data_size_

private

Definition at line 155 of file CsvReader.h.

Referenced by checkForMoreRows(), getRemainingSize(), serialize(), and SingleFileReader().

std::FILE* foreign_storage::SingleFileReader::file_

private

Definition at line 153 of file CsvReader.h.

Referenced by checkForMoreRows(), read(), readRegion(), SingleFileReader(), and ~SingleFileReader().

size_t foreign_storage::SingleFileReader::header_offset_

private

Definition at line 160 of file CsvReader.h.

Referenced by checkForMoreRows(), readRegion(), serialize(), and SingleFileReader().

bool foreign_storage::SingleFileReader::scan_finished_

private

Definition at line 157 of file CsvReader.h.

Referenced by checkForMoreRows(), isScanFinished(), read(), and serialize().

size_t foreign_storage::SingleFileReader::total_bytes_read_

private

Definition at line 162 of file CsvReader.h.

Referenced by checkForMoreRows(), getRemainingSize(), read(), serialize(), and SingleFileReader().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/CsvReader.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/CsvReader.cpp

Public Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation