OmniSciDB
94e8789169
|
Represents/provides access to contiguous data stored in the file system. More...
#include <FileBuffer.h>
Public Member Functions | |
FileBuffer (FileMgr *fm, const size_t pageSize, const ChunkKey &chunkKey, const size_t initialSize=0) | |
Constructs a FileBuffer object. More... | |
FileBuffer (FileMgr *fm, const size_t pageSize, const ChunkKey &chunkKey, const SQLTypeInfo sqlType, const size_t initialSize=0) | |
FileBuffer (FileMgr *fm, const ChunkKey &chunkKey, const std::vector< HeaderInfo >::const_iterator &headerStartIt, const std::vector< HeaderInfo >::const_iterator &headerEndIt) | |
~FileBuffer () override | |
Destructor. More... | |
Page | addNewMultiPage (const int32_t epoch) |
void | reserve (const size_t numBytes) override |
void | freeMetadataPages () |
size_t | freeChunkPages () |
void | freePages () |
void | freePagesBeforeEpoch (const int32_t targetEpoch) |
void | read (int8_t *const dst, const size_t numBytes=0, const size_t offset=0, const MemoryLevel dstMemoryLevel=CPU_LEVEL, const int32_t deviceId=-1) override |
void | write (int8_t *src, const size_t numBytes, const size_t offset=0, const MemoryLevel srcMemoryLevel=CPU_LEVEL, const int32_t deviceId=-1) override |
Writes the contents of source (src) into new versions of the affected logical pages. More... | |
void | append (int8_t *src, const size_t numBytes, const MemoryLevel srcMemoryLevel=CPU_LEVEL, const int32_t deviceId=-1) override |
void | copyPage (Page &srcPage, Page &destPage, const size_t numBytes, const size_t offset=0) |
Data_Namespace::MemoryLevel | getType () const override |
int8_t * | getMemoryPtr () override |
Not implemented for FileMgr – throws a runtime_error. More... | |
size_t | pageCount () const override |
Returns the number of pages in the FileBuffer. More... | |
size_t | pageSize () const override |
Returns the size in bytes of each page in the FileBuffer. More... | |
virtual size_t | pageDataSize () const |
Returns the size in bytes of the data portion of each page in the FileBuffer. More... | |
virtual size_t | reservedHeaderSize () const |
virtual std::vector< MultiPage > | getMultiPage () const |
Returns vector of MultiPages in the FileBuffer. More... | |
size_t | reservedSize () const override |
Returns the total number of bytes allocated for the FileBuffer. More... | |
size_t | numMetadataPages () const |
Returns the total number of used bytes in the FileBuffer. More... | |
![]() | |
AbstractBuffer (const int device_id) | |
AbstractBuffer (const int device_id, const SQLTypeInfo sql_type) | |
virtual | ~AbstractBuffer () |
virtual void | read (int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0 |
virtual void | write (int8_t *src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int src_device_id=-1)=0 |
virtual void | append (int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0 |
virtual void | setMemoryPtr (int8_t *new_ptr) |
virtual int | pin () |
virtual int | unPin () |
virtual int | getPinCount () |
size_t | size () const |
int | getDeviceId () const |
bool | isDirty () const |
bool | isAppended () const |
bool | isUpdated () const |
bool | hasEncoder () const |
SQLTypeInfo | getSqlType () const |
void | setSqlType (const SQLTypeInfo &sql_type) |
Encoder * | getEncoder () const |
void | setDirty () |
void | setUpdated () |
void | setAppended () |
void | setSize (const size_t size) |
void | clearDirtyBits () |
void | initEncoder (const SQLTypeInfo &tmp_sql_type) |
void | syncEncoder (const AbstractBuffer *src_buffer) |
void | copyTo (AbstractBuffer *destination_buffer, const size_t num_bytes=0) |
void | resetToEmpty () |
Private Member Functions | |
void | writeHeader (Page &page, const int32_t pageId, const int32_t epoch, const bool writeMetadata=false) |
Write header writes header at top of page in format. More... | |
void | writeMetadata (const int32_t epoch) |
void | readMetadata (const Page &page) |
void | calcHeaderBuffer () |
void | freePage (const Page &page, const bool isRolloff) |
void | freePagesBeforeEpochForMultiPage (MultiPage &multiPage, const int32_t targetEpoch, const int32_t currentEpoch) |
Private Attributes | |
FileMgr * | fm_ |
MultiPage | metadataPages_ |
std::vector< MultiPage > | multiPages_ |
size_t | pageSize_ |
size_t | pageDataSize_ |
size_t | reservedHeaderSize_ |
ChunkKey | chunkKey_ |
Static Private Attributes | |
static size_t | headerBufferOffset_ = 32 |
Friends | |
class | FileMgr |
Additional Inherited Members | |
![]() | |
std::unique_ptr< Encoder > | encoder_ |
SQLTypeInfo | sql_type_ |
size_t | size_ |
int | device_id_ |
Represents/provides access to contiguous data stored in the file system.
The FileBuffer consists of logical pages, which can map to any identically-sized page in any file of the underlying file system. A page's metadata (file and page number) are stored in MultiPage objects, and each MultiPage includes page metadata for multiple versions of the same page.
Note that a "Chunk" is brought into a FileBuffer by the FileMgr.
Note(s): Forbid Copying Idiom 4.1
Definition at line 56 of file FileBuffer.h.
File_Namespace::FileBuffer::FileBuffer | ( | FileMgr * | fm, |
const size_t | pageSize, | ||
const ChunkKey & | chunkKey, | ||
const size_t | initialSize = 0 |
||
) |
Constructs a FileBuffer object.
Definition at line 39 of file FileBuffer.cpp.
References calcHeaderBuffer(), CHECK, fm_, pageDataSize_, pageSize_, and reservedHeaderSize_.
File_Namespace::FileBuffer::FileBuffer | ( | FileMgr * | fm, |
const size_t | pageSize, | ||
const ChunkKey & | chunkKey, | ||
const SQLTypeInfo | sqlType, | ||
const size_t | initialSize = 0 |
||
) |
Definition at line 67 of file FileBuffer.cpp.
References calcHeaderBuffer(), CHECK, fm_, pageDataSize_, pageSize_, and reservedHeaderSize_.
File_Namespace::FileBuffer::FileBuffer | ( | FileMgr * | fm, |
const ChunkKey & | chunkKey, | ||
const std::vector< HeaderInfo >::const_iterator & | headerStartIt, | ||
const std::vector< HeaderInfo >::const_iterator & | headerEndIt | ||
) |
Definition at line 82 of file FileBuffer.cpp.
References calcHeaderBuffer(), CHECK, File_Namespace::MultiPage::current(), logger::FATAL, File_Namespace::Page::fileId, fm_, LOG, metadataPages_, multiPages_, File_Namespace::EpochedPage::page, pageDataSize_, pageSize_, File_Namespace::MultiPage::push(), readMetadata(), reservedHeaderSize_, and show_chunk().
|
override |
Destructor.
Definition at line 130 of file FileBuffer.cpp.
Page File_Namespace::FileBuffer::addNewMultiPage | ( | const int32_t | epoch | ) |
Definition at line 388 of file FileBuffer.cpp.
References fm_, multiPages_, pageSize_, File_Namespace::MultiPage::push(), and File_Namespace::FileMgr::requestFreePage().
Referenced by append(), reserve(), and write().
|
override |
Definition at line 472 of file FileBuffer.cpp.
References addNewMultiPage(), CHECK, File_Namespace::FileMgr::epoch(), File_Namespace::Page::fileId, fm_, File_Namespace::FileMgr::getFileInfoForFileId(), multiPages_, pageDataSize_, File_Namespace::Page::pageNum, pageSize_, reservedHeaderSize_, Data_Namespace::AbstractBuffer::setAppended(), Data_Namespace::AbstractBuffer::size_, File_Namespace::FileInfo::write(), and writeHeader().
Referenced by File_Namespace::FileMgr::putBuffer().
|
private |
Definition at line 146 of file FileBuffer.cpp.
References chunkKey_, headerBufferOffset_, and reservedHeaderSize_.
Referenced by FileBuffer().
void File_Namespace::FileBuffer::copyPage | ( | Page & | srcPage, |
Page & | destPage, | ||
const size_t | numBytes, | ||
const size_t | offset = 0 |
||
) |
Definition at line 368 of file FileBuffer.cpp.
References CHECK, checked_malloc(), File_Namespace::Page::fileId, fm_, File_Namespace::FileMgr::getFileInfoForFileId(), pageDataSize_, File_Namespace::Page::pageNum, pageSize_, File_Namespace::FileInfo::read(), reservedHeaderSize_, and File_Namespace::FileInfo::write().
Referenced by write().
size_t File_Namespace::FileBuffer::freeChunkPages | ( | ) |
Definition at line 172 of file FileBuffer.cpp.
References freePage(), and multiPages_.
Referenced by foreign_storage::ForeignStorageCache::eraseChunk(), foreign_storage::ForeignStorageCache::evictChunkByIterator(), and freePages().
void File_Namespace::FileBuffer::freeMetadataPages | ( | ) |
Definition at line 161 of file FileBuffer.cpp.
References freePage(), metadataPages_, File_Namespace::MultiPage::pageVersions, and File_Namespace::MultiPage::pop().
Referenced by freePages().
|
private |
Definition at line 156 of file FileBuffer.cpp.
References File_Namespace::Page::fileId, fm_, File_Namespace::FileInfo::freePage(), File_Namespace::FileMgr::getFileInfoForFileId(), and File_Namespace::Page::pageNum.
Referenced by freeChunkPages(), freeMetadataPages(), and freePagesBeforeEpochForMultiPage().
void File_Namespace::FileBuffer::freePages | ( | ) |
Definition at line 186 of file FileBuffer.cpp.
References freeChunkPages(), and freeMetadataPages().
void File_Namespace::FileBuffer::freePagesBeforeEpoch | ( | const int32_t | targetEpoch | ) |
Definition at line 201 of file FileBuffer.cpp.
References CHECK_LE, File_Namespace::FileMgr::epoch(), fm_, freePagesBeforeEpochForMultiPage(), metadataPages_, and multiPages_.
|
private |
Definition at line 191 of file FileBuffer.cpp.
References freePage(), and File_Namespace::MultiPage::freePagesBeforeEpoch().
Referenced by freePagesBeforeEpoch().
|
inlineoverridevirtual |
Not implemented for FileMgr – throws a runtime_error.
Implements Data_Namespace::AbstractBuffer.
Definition at line 122 of file FileBuffer.h.
References logger::FATAL, and LOG.
|
inlinevirtual |
Returns vector of MultiPages in the FileBuffer.
Definition at line 141 of file FileBuffer.h.
Referenced by File_Namespace::FileMgr::init(), and read().
|
inlineoverridevirtual |
Implements Data_Namespace::AbstractBuffer.
Definition at line 119 of file FileBuffer.h.
References Data_Namespace::DISK_LEVEL.
|
inline |
Returns the total number of used bytes in the FileBuffer.
Definition at line 149 of file FileBuffer.h.
|
inlineoverridevirtual |
Returns the number of pages in the FileBuffer.
Implements Data_Namespace::AbstractBuffer.
Definition at line 128 of file FileBuffer.h.
|
inlinevirtual |
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition at line 134 of file FileBuffer.h.
Referenced by File_Namespace::FileMgr::init(), and File_Namespace::readForThread().
|
inlineoverridevirtual |
Returns the size in bytes of each page in the FileBuffer.
Implements Data_Namespace::AbstractBuffer.
Definition at line 131 of file FileBuffer.h.
Referenced by File_Namespace::FileMgr::init(), File_Namespace::readForThread(), and writeHeader().
|
override |
Definition at line 265 of file FileBuffer.cpp.
References CHECK, Data_Namespace::CPU_LEVEL, logger::FATAL, fm_, getMultiPage(), File_Namespace::FileMgr::getNumReaderThreads(), generate_TableFunctionsFactory_init::i, LOG, File_Namespace::readThreadDS::multiPages, multiPages_, pageDataSize_, File_Namespace::readForThread(), File_Namespace::readThreadDS::t_bytesLeft, File_Namespace::readThreadDS::t_curPtr, File_Namespace::readThreadDS::t_endPage, File_Namespace::readThreadDS::t_fm, File_Namespace::readThreadDS::t_isFirstPage, File_Namespace::readThreadDS::t_startPage, and File_Namespace::readThreadDS::t_startPageOffset.
|
private |
Definition at line 415 of file FileBuffer.cpp.
References CHECK, Data_Namespace::AbstractBuffer::encoder_, generate_TableFunctionsFactory_init::f, File_Namespace::Page::fileId, fm_, File_Namespace::FileMgr::getFileForFileId(), Data_Namespace::AbstractBuffer::initEncoder(), METADATA_PAGE_SIZE, METADATA_VERSION, NUM_METADATA, File_Namespace::Page::pageNum, pageSize_, reservedHeaderSize_, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), Data_Namespace::AbstractBuffer::size_, Data_Namespace::AbstractBuffer::sql_type_, and setup::version.
Referenced by FileBuffer().
|
overridevirtual |
Implements Data_Namespace::AbstractBuffer.
Definition at line 135 of file FileBuffer.cpp.
References addNewMultiPage(), File_Namespace::FileMgr::epoch(), fm_, multiPages_, pageSize_, and writeHeader().
|
inlinevirtual |
Returns the size in bytes of the reserved header portion of each page in the FileBuffer.
Definition at line 138 of file FileBuffer.h.
Referenced by File_Namespace::FileMgr::init(), and File_Namespace::readForThread().
|
inlineoverridevirtual |
Returns the total number of bytes allocated for the FileBuffer.
Implements Data_Namespace::AbstractBuffer.
Definition at line 144 of file FileBuffer.h.
|
override |
Writes the contents of source (src) into new versions of the affected logical pages.
This method will write the contents of source (src) into new version of the affected logical pages. New pages are only appended if the value of epoch (in FileMgr)
Definition at line 516 of file FileBuffer.cpp.
References addNewMultiPage(), CHECK, copyPage(), Data_Namespace::CPU_LEVEL, File_Namespace::FileMgr::epoch(), logger::FATAL, File_Namespace::Page::fileId, fm_, File_Namespace::FileMgr::getFileInfoForFileId(), LOG, multiPages_, pageDataSize_, File_Namespace::Page::pageNum, pageSize_, File_Namespace::FileMgr::requestFreePage(), reservedHeaderSize_, Data_Namespace::AbstractBuffer::setAppended(), Data_Namespace::AbstractBuffer::setDirty(), Data_Namespace::AbstractBuffer::setUpdated(), Data_Namespace::AbstractBuffer::size_, File_Namespace::FileInfo::write(), and writeHeader().
Referenced by File_Namespace::FileMgr::putBuffer().
|
private |
Write header writes header at top of page in format.
Definition at line 396 of file FileBuffer.cpp.
References chunkKey_, gpu_enabled::copy(), File_Namespace::Page::fileId, fm_, File_Namespace::FileMgr::getFileInfoForFileId(), METADATA_PAGE_SIZE, File_Namespace::Page::pageNum, pageSize(), pageSize_, and File_Namespace::FileInfo::write().
Referenced by append(), File_Namespace::FileMgr::init(), reserve(), write(), and writeMetadata().
|
private |
Definition at line 441 of file FileBuffer.cpp.
References Data_Namespace::AbstractBuffer::encoder_, generate_TableFunctionsFactory_init::f, File_Namespace::Page::fileId, fm_, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), File_Namespace::FileMgr::getFileForFileId(), Data_Namespace::AbstractBuffer::hasEncoder(), METADATA_PAGE_SIZE, METADATA_VERSION, metadataPages_, NUM_METADATA, File_Namespace::Page::pageNum, pageSize_, File_Namespace::MultiPage::push(), File_Namespace::FileMgr::requestFreePage(), reservedHeaderSize_, Data_Namespace::AbstractBuffer::size_, Data_Namespace::AbstractBuffer::sql_type_, and writeHeader().
|
friend |
Definition at line 57 of file FileBuffer.h.
|
private |
Definition at line 180 of file FileBuffer.h.
Referenced by calcHeaderBuffer(), and writeHeader().
|
private |
Definition at line 172 of file FileBuffer.h.
Referenced by addNewMultiPage(), append(), copyPage(), FileBuffer(), freePage(), freePagesBeforeEpoch(), read(), readMetadata(), reserve(), write(), writeHeader(), and writeMetadata().
|
staticprivate |
Definition at line 174 of file FileBuffer.h.
Referenced by calcHeaderBuffer().
|
private |
Definition at line 175 of file FileBuffer.h.
Referenced by FileBuffer(), freeMetadataPages(), freePagesBeforeEpoch(), and writeMetadata().
|
private |
Definition at line 176 of file FileBuffer.h.
Referenced by addNewMultiPage(), append(), FileBuffer(), freeChunkPages(), freePagesBeforeEpoch(), File_Namespace::FileMgr::init(), read(), reserve(), and write().
|
private |
Definition at line 178 of file FileBuffer.h.
Referenced by append(), copyPage(), FileBuffer(), read(), and write().
|
private |
Definition at line 177 of file FileBuffer.h.
Referenced by addNewMultiPage(), append(), copyPage(), FileBuffer(), readMetadata(), reserve(), write(), writeHeader(), and writeMetadata().
|
private |
Definition at line 179 of file FileBuffer.h.
Referenced by append(), calcHeaderBuffer(), copyPage(), FileBuffer(), readMetadata(), write(), and writeMetadata().