OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::SingleTextFileReader Class Reference

#include <FileReader.h>

+ Inheritance diagram for foreign_storage::SingleTextFileReader:
+ Collaboration diagram for foreign_storage::SingleTextFileReader:

Public Member Functions

 SingleTextFileReader (const std::string &file_path, const import_export::CopyParams &copy_params)
 
 SingleTextFileReader (const std::string &file_path, const import_export::CopyParams &copy_params, const rapidjson::Value &value)
 
 ~SingleTextFileReader () override
 
 SingleTextFileReader (const SingleTextFileReader &)=delete
 
SingleTextFileReaderoperator= (const SingleTextFileReader &)=delete
 
size_t read (void *buffer, size_t max_size) override
 
size_t readRegion (void *buffer, size_t offset, size_t size) override
 
bool isScanFinished () override
 
size_t getRemainingSize () override
 
bool isRemainingSizeKnown () override
 
void checkForMoreRows (size_t file_offset, const ForeignServer *server_options, const UserMapping *user_mapping) override
 
void serialize (rapidjson::Value &value, rapidjson::Document::AllocatorType &allocator) const override
 
- Public Member Functions inherited from foreign_storage::SingleFileReader
 SingleFileReader (const std::string &file_path, const import_export::CopyParams &copy_params)
 
 ~SingleFileReader () override=default
 
FirstLineByFilePath getFirstLineForEachFile () const override
 
bool isEndOfLastFile () override
 
- Public Member Functions inherited from foreign_storage::FileReader
 FileReader (const std::string &file_path, const import_export::CopyParams &copy_params)
 
virtual ~FileReader ()=default
 

Private Member Functions

std::string getFirstLine () const override
 
void skipHeader () override
 

Private Attributes

std::FILE * file_
 
size_t data_size_
 
bool scan_finished_
 
size_t header_offset_
 
size_t total_bytes_read_
 

Additional Inherited Members

- Protected Attributes inherited from foreign_storage::FileReader
import_export::CopyParams copy_params_
 
std::string file_path_
 
- Static Protected Attributes inherited from foreign_storage::SingleFileReader
static constexpr size_t DEFAULT_HEADER_READ_SIZE {1024}
 

Detailed Description

Definition at line 141 of file FileReader.h.

Constructor & Destructor Documentation

foreign_storage::SingleTextFileReader::SingleTextFileReader ( const std::string &  file_path,
const import_export::CopyParams copy_params 
)

Definition at line 84 of file FileReader.cpp.

References data_size_, file_, omnisci::fopen(), foreign_storage::anonymous_namespace{FileReader.cpp}::get_data_size(), header_offset_, and skipHeader().

86  : SingleFileReader(file_path, copy_params)
87  , scan_finished_(false)
88  , header_offset_(0)
89  , total_bytes_read_(0) {
90  file_ = fopen(file_path.c_str(), "rb");
91  if (!file_) {
92  throw std::runtime_error{"An error occurred when attempting to open file \"" +
93  file_path + "\". " + strerror(errno)};
94  }
95 
96  // Skip header and record offset
97  skipHeader();
98  fseek(file_, 0, SEEK_END);
99 
101 
102  if (fseek(file_, static_cast<long int>(header_offset_), SEEK_SET) != 0) {
103  throw std::runtime_error{"An error occurred when attempting to open file \"" +
104  file_path + "\". " + strerror(errno)};
105  };
106 }
::FILE * fopen(const char *filename, const char *mode)
Definition: omnisci_fs.cpp:72
size_t get_data_size(size_t file_size, size_t header_size)
Definition: FileReader.cpp:65
SingleFileReader(const std::string &file_path, const import_export::CopyParams &copy_params)
Definition: FileReader.cpp:72

+ Here is the call graph for this function:

foreign_storage::SingleTextFileReader::SingleTextFileReader ( const std::string &  file_path,
const import_export::CopyParams copy_params,
const rapidjson::Value &  value 
)

Definition at line 108 of file FileReader.cpp.

References data_size_, file_, omnisci::fopen(), foreign_storage::json_utils::get_value_from_object(), header_offset_, and total_bytes_read_.

111  : SingleFileReader(file_path, copy_params)
112  , scan_finished_(true)
113  , header_offset_(0)
114  , total_bytes_read_(0) {
115  file_ = fopen(file_path.c_str(), "rb");
116  if (!file_) {
117  throw std::runtime_error{"An error occurred when attempting to open file \"" +
118  file_path + "\". " + strerror(errno)};
119  }
120  json_utils::get_value_from_object(value, header_offset_, "header_offset");
121  json_utils::get_value_from_object(value, total_bytes_read_, "total_bytes_read");
122  json_utils::get_value_from_object(value, data_size_, "data_size");
123 }
::FILE * fopen(const char *filename, const char *mode)
Definition: omnisci_fs.cpp:72
void get_value_from_object(const rapidjson::Value &object, T &value, const std::string &name)
Definition: FsiJsonUtils.h:164
SingleFileReader(const std::string &file_path, const import_export::CopyParams &copy_params)
Definition: FileReader.cpp:72

+ Here is the call graph for this function:

foreign_storage::SingleTextFileReader::~SingleTextFileReader ( )
inlineoverride

Definition at line 148 of file FileReader.h.

References file_.

148 { fclose(file_); }
foreign_storage::SingleTextFileReader::SingleTextFileReader ( const SingleTextFileReader )
delete

Member Function Documentation

void foreign_storage::SingleTextFileReader::checkForMoreRows ( size_t  file_offset,
const ForeignServer server_options,
const UserMapping user_mapping 
)
overridevirtual

Rescan the target files Throws an exception if the rescan fails (ie files are not in a valid appended state or not supported)

Parameters
file_offset- where to resume the scan from (end of the last row) as not all of the bytes may have been consumed by the upstream compoennet
server_options- only needed for S3 backed files
user_mapping- only needed for S3 backed files

Reimplemented from foreign_storage::FileReader.

Definition at line 135 of file FileReader.cpp.

References CHECK, data_size_, file_, foreign_storage::FileReader::file_path_, omnisci::fopen(), foreign_storage::anonymous_namespace{FileReader.cpp}::get_data_size(), header_offset_, isScanFinished(), scan_finished_, foreign_storage::throw_removed_row_error(), to_string(), and total_bytes_read_.

137  {
139  // Re-open file and check if there is any new data in it
140  fclose(file_);
141  file_ = fopen(file_path_.c_str(), "rb");
142  if (!file_) {
143  throw std::runtime_error{"An error occurred when attempting to open file \"" +
144  file_path_ + "\". " + strerror(errno)};
145  }
146  fseek(file_, 0, SEEK_END);
147  size_t new_file_size = ftell(file_);
148  size_t new_data_size = get_data_size(new_file_size, header_offset_);
149  if (new_data_size < data_size_) {
151  }
152  if (fseek(file_, static_cast<long int>(file_offset + header_offset_), SEEK_SET) != 0) {
153  throw std::runtime_error{"An error occurred when attempting to read offset " +
154  std::to_string(file_offset + header_offset_) +
155  " in file: \"" + file_path_ + "\". " + strerror(errno)};
156  }
157  if (new_data_size > data_size_) {
158  scan_finished_ = false;
159  total_bytes_read_ = file_offset;
160  data_size_ = new_data_size;
161  }
162 }
::FILE * fopen(const char *filename, const char *mode)
Definition: omnisci_fs.cpp:72
std::string to_string(char const *&&v)
void throw_removed_row_error(const std::string &file_path)
size_t get_data_size(size_t file_size, size_t header_size)
Definition: FileReader.cpp:65
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

std::string foreign_storage::SingleTextFileReader::getFirstLine ( ) const
overrideprivatevirtual

Implements foreign_storage::SingleFileReader.

Definition at line 170 of file FileReader.cpp.

References CHECK, foreign_storage::FileReader::copy_params_, foreign_storage::FileReader::file_path_, parse_ast::line, and import_export::CopyParams::line_delim.

Referenced by skipHeader().

170  {
171  std::ifstream file{file_path_};
172  CHECK(file.good());
173  std::string line;
174  std::getline(file, line, copy_params_.line_delim);
175  file.close();
176  return line;
177 }
import_export::CopyParams copy_params_
Definition: FileReader.h:118
tuple line
Definition: parse_ast.py:10
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the caller graph for this function:

size_t foreign_storage::SingleTextFileReader::getRemainingSize ( )
inlineoverridevirtual
Returns
size of the remaining content to be read

Implements foreign_storage::FileReader.

Definition at line 176 of file FileReader.h.

References data_size_, and total_bytes_read_.

bool foreign_storage::SingleTextFileReader::isRemainingSizeKnown ( )
inlineoverridevirtual
Returns
if remaining size is known

Implements foreign_storage::FileReader.

Definition at line 178 of file FileReader.h.

178 { return true; };
bool foreign_storage::SingleTextFileReader::isScanFinished ( )
inlineoverridevirtual
Returns
true if the entire file has been read

Implements foreign_storage::FileReader.

Definition at line 174 of file FileReader.h.

References scan_finished_.

Referenced by checkForMoreRows(), and readRegion().

+ Here is the caller graph for this function:

SingleTextFileReader& foreign_storage::SingleTextFileReader::operator= ( const SingleTextFileReader )
delete
size_t foreign_storage::SingleTextFileReader::read ( void *  buffer,
size_t  max_size 
)
inlineoverridevirtual

Read up to max_size bytes from archive into buffer starting starting from the end of the last read

Parameters
buffer- buffer to load into
max_size- maximum number of bytes to read into the buffer
Returns
number of bytes actually read

Implements foreign_storage::FileReader.

Definition at line 154 of file FileReader.h.

References file_, scan_finished_, and total_bytes_read_.

154  {
155  size_t bytes_read = fread(buffer, 1, max_size, file_);
156  if (!scan_finished_) {
157  scan_finished_ = feof(file_);
158  }
159 
160  total_bytes_read_ += bytes_read;
161  return bytes_read;
162  }
size_t foreign_storage::SingleTextFileReader::readRegion ( void *  buffer,
size_t  offset,
size_t  size 
)
inlineoverridevirtual

Read up to max_size bytes from archive, starting at given offset isScanFinished() must return true to use readRegion

Parameters
buffer- buffer to load into
offset- starting point into the archive to read
size- maximum number of bytes to read into the buffer
Returns
number of bytes actually read

Implements foreign_storage::FileReader.

Definition at line 164 of file FileReader.h.

References CHECK, file_, foreign_storage::FileReader::file_path_, header_offset_, isScanFinished(), and to_string().

164  {
166  if (fseek(file_, static_cast<long int>(offset + header_offset_), SEEK_SET) != 0) {
167  throw std::runtime_error{"An error occurred when attempting to read offset " +
168  std::to_string(offset) + " in file: \"" + file_path_ +
169  "\". " + strerror(errno)};
170  }
171  return fread(buffer, 1, size, file_);
172  }
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

void foreign_storage::SingleTextFileReader::serialize ( rapidjson::Value &  value,
rapidjson::Document::AllocatorType &  allocator 
) const
overridevirtual

Serialize internal state to given json object This Json will later be used to restore the reader state through a constructor must be called when isScanFinished() is true

Parameters
value- json object to store needed state to this function can store any needed data or none
allocator- allocator to use for json contruction

Implements foreign_storage::FileReader.

Definition at line 125 of file FileReader.cpp.

References foreign_storage::json_utils::add_value_to_object(), CHECK, data_size_, header_offset_, scan_finished_, and total_bytes_read_.

127  {
129  json_utils::add_value_to_object(value, header_offset_, "header_offset", allocator);
131  value, total_bytes_read_, "total_bytes_read", allocator);
132  json_utils::add_value_to_object(value, data_size_, "data_size", allocator);
133 }
void add_value_to_object(rapidjson::Value &object, const T &value, const std::string &name, rapidjson::Document::AllocatorType &allocator)
Definition: FsiJsonUtils.h:149
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

void foreign_storage::SingleTextFileReader::skipHeader ( )
overrideprivatevirtual

Implements foreign_storage::SingleFileReader.

Definition at line 164 of file FileReader.cpp.

References foreign_storage::FileReader::copy_params_, getFirstLine(), import_export::CopyParams::has_header, header_offset_, and import_export::NO_HEADER.

Referenced by SingleTextFileReader().

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

size_t foreign_storage::SingleTextFileReader::data_size_
private

Definition at line 192 of file FileReader.h.

Referenced by checkForMoreRows(), getRemainingSize(), serialize(), and SingleTextFileReader().

std::FILE* foreign_storage::SingleTextFileReader::file_
private
size_t foreign_storage::SingleTextFileReader::header_offset_
private
bool foreign_storage::SingleTextFileReader::scan_finished_
private

Definition at line 194 of file FileReader.h.

Referenced by checkForMoreRows(), isScanFinished(), read(), and serialize().

size_t foreign_storage::SingleTextFileReader::total_bytes_read_
private

The documentation for this class was generated from the following files: