OmniSciDB
94e8789169
|
#include <FileMgr.h>
Public Member Functions | |
FileMgr (const int32_t deviceId, GlobalFileMgr *gfm, const std::pair< const int32_t, const int32_t > fileMgrKey, const int32_t max_rollback_epochs=-1, const size_t num_reader_threads=0, const int32_t epoch=-1, const size_t defaultPageSize=DEFAULT_PAGE_SIZE) | |
Constructor. More... | |
FileMgr (const int32_t deviceId, GlobalFileMgr *gfm, const std::pair< const int32_t, const int32_t > fileMgrKey, const size_t defaultPageSize, const bool runCoreInit) | |
FileMgr (GlobalFileMgr *gfm, const size_t defaultPageSize, std::string basePath) | |
~FileMgr () override | |
Destructor. More... | |
StorageStats | getStorageStats () |
FileBuffer * | createBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override |
Creates a chunk with the specified key and page size. More... | |
bool | isBufferOnDevice (const ChunkKey &key) override |
void | deleteBuffer (const ChunkKey &key, const bool purge=true) override |
Deletes the chunk with the specified key. More... | |
void | deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override |
FileBuffer * | getBuffer (const ChunkKey &key, const size_t numBytes=0) override |
Returns the a pointer to the chunk with the specified key. More... | |
void | fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override |
FileBuffer * | putBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override |
Puts the contents of d into the Chunk with the given key. More... | |
AbstractBuffer * | alloc (const size_t numBytes) override |
void | free (AbstractBuffer *buffer) override |
Page | requestFreePage (size_t pagesize, const bool isMetadata) |
MgrType | getMgrType () override |
std::string | getStringMgrType () override |
std::string | printSlabs () override |
void | clearSlabs () override |
size_t | getMaxSize () override |
size_t | getInUseSize () override |
size_t | getAllocated () override |
bool | isAllocationCapped () override |
FileInfo * | getFileInfoForFileId (const int32_t fileId) |
uint64_t | getTotalFileSize () const |
FileMetadata | getMetadataForFile (const boost::filesystem::directory_iterator &fileIterator) |
void | init (const size_t num_reader_threads, const int32_t epochOverride) |
void | init (const std::string &dataPathToConvertFrom, const int32_t epochOverride) |
bool | coreInit () |
Determines file path, and if exists, runs file migration and opens and reads epoch file. More... | |
void | copyPage (Page &srcPage, FileMgr *destFileMgr, Page &destPage, const size_t reservedHeaderSize, const size_t numBytes, const size_t offset) |
void | requestFreePages (size_t npages, size_t pagesize, std::vector< Page > &pages, const bool isMetadata) |
Obtains free pages – creates new files if necessary – of the requested size. More... | |
void | getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override |
void | checkpoint () override |
Fsyncs data files, writes out epoch and fsyncs that. More... | |
void | checkpoint (const int32_t db_id, const int32_t tb_id) override |
int32_t | epoch () |
Returns current value of epoch - should be one greater than recorded at last checkpoint. More... | |
int32_t | epochFloor () |
int32_t | incrementEpoch () |
int32_t | lastCheckpointedEpoch () |
Returns value of epoch at last checkpoint. More... | |
int32_t | maxRollbackEpochs () |
Returns value max_rollback_epochs. More... | |
size_t | getNumReaderThreads () |
Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More... | |
FILE * | getFileForFileId (const int32_t fileId) |
Returns FILE pointer associated with requested fileId. More... | |
size_t | getNumChunks () override |
size_t | getNumUsedPages () const |
size_t | getNumUsedMetadataPages () const |
size_t | getNumUsedMetadataPagesForChunkKey (const ChunkKey &chunkKey) const |
int32_t | getDBVersion () const |
Index for looking up chunks. More... | |
bool | getDBConvert () const |
void | createTopLevelMetadata () |
std::string | getFileMgrBasePath () const |
void | closeRemovePhysical () |
void | removeTableRelatedDS (const int32_t db_id, const int32_t table_id) override |
void | free_page (std::pair< FileInfo *, int32_t > &&page) |
const std::pair< const int32_t, const int32_t > | get_fileMgrKey () const |
Public Attributes | |
ChunkKeyToChunkMap | chunkIndex_ |
Protected Member Functions | |
FileMgr (const int epoch) | |
Private Member Functions | |
FileInfo * | createFile (const size_t pageSize, const size_t numPages) |
Adds a file to the file manager repository. More... | |
FileInfo * | openExistingFile (const std::string &path, const int32_t fileId, const size_t pageSize, const size_t numPages, std::vector< HeaderInfo > &headerVec) |
void | createEpochFile (const std::string &epochFileName) |
int32_t | openAndReadLegacyEpochFile (const std::string &epochFileName) |
void | openAndReadEpochFile (const std::string &epochFileName) |
void | writeAndSyncEpochToDisk () |
void | setEpoch (const int32_t newEpoch) |
void | freePagesBeforeEpoch (const int32_t minRollbackEpoch) |
void | rollOffOldData (const int32_t epochCeiling, const bool shouldCheckpoint) |
int32_t | readVersionFromDisk (const std::string &versionFileName) const |
void | writeAndSyncVersionToDisk (const std::string &versionFileName, const int32_t version) |
void | processFileFutures (std::vector< std::future< std::vector< HeaderInfo >>> &file_futures, std::vector< HeaderInfo > &headerVec) |
FileBuffer * | createBufferUnlocked (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) |
void | migrateToLatestFileMgrVersion () |
void | migrateEpochFileV0 () |
Private Attributes | |
GlobalFileMgr * | gfm_ |
std::pair< const int32_t, const int32_t > | fileMgrKey_ |
Global FileMgr. More... | |
int32_t | maxRollbackEpochs_ |
std::string | fileMgrBasePath_ |
std::vector< FileInfo * > | files_ |
PageSizeFileMMap | fileIndex_ |
A vector of files accessible via a file identifier. More... | |
size_t | num_reader_threads_ |
Maps page sizes to FileInfo objects. More... | |
size_t | defaultPageSize_ |
number of threads used when loading data More... | |
unsigned | nextFileId_ |
Epoch | epoch_ |
the index of the next file id More... | |
bool | epochIsCheckpointed_ = true |
FILE * | epochFile_ = nullptr |
int32_t | db_version_ |
int32_t | fileMgrVersion_ |
const int32_t | latestFileMgrVersion_ {1} |
FILE * | DBMetaFile_ = nullptr |
std::mutex | getPageMutex_ |
pointer to DB level metadata More... | |
mapd_shared_mutex | chunkIndexMutex_ |
mapd_shared_mutex | files_rw_mutex_ |
mapd_shared_mutex | mutex_free_page_ |
std::vector< std::pair < FileInfo *, int32_t > > | free_pages_ |
bool | isFullyInitted_ {false} |
Friends | |
class | GlobalFileMgr |
File_Namespace::FileMgr::FileMgr | ( | const int32_t | deviceId, |
GlobalFileMgr * | gfm, | ||
const std::pair< const int32_t, const int32_t > | fileMgrKey, | ||
const int32_t | max_rollback_epochs = -1 , |
||
const size_t | num_reader_threads = 0 , |
||
const int32_t | epoch = -1 , |
||
const size_t | defaultPageSize = DEFAULT_PAGE_SIZE |
||
) |
Constructor.
File_Namespace::FileMgr::FileMgr | ( | const int32_t | deviceId, |
GlobalFileMgr * | gfm, | ||
const std::pair< const int32_t, const int32_t > | fileMgrKey, | ||
const size_t | defaultPageSize, | ||
const bool | runCoreInit | ||
) |
Definition at line 82 of file FileMgr.cpp.
References coreInit(), epochFile_, fileMgrBasePath_, fileMgrKey_, files_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, and to_string().
File_Namespace::FileMgr::FileMgr | ( | GlobalFileMgr * | gfm, |
const size_t | defaultPageSize, | ||
std::string | basePath | ||
) |
Definition at line 105 of file FileMgr.cpp.
References init().
|
override |
Destructor.
Definition at line 121 of file FileMgr.cpp.
References chunkIndex_, File_Namespace::close(), DBMetaFile_, epochFile_, and files_.
|
protected |
Definition at line 117 of file FileMgr.cpp.
References Epoch::ceiling(), and epoch_.
|
override |
Definition at line 862 of file FileMgr.cpp.
References logger::FATAL, and LOG.
|
override |
Fsyncs data files, writes out epoch and fsyncs that.
Definition at line 666 of file FileMgr.cpp.
References chunkIndex_, chunkIndexMutex_, epoch(), logger::FATAL, fileMgrKey_, files_, files_rw_mutex_, free_page(), free_pages_, incrementEpoch(), lastCheckpointedEpoch(), LOG, mutex_free_page_, rollOffOldData(), VLOG, and writeAndSyncEpochToDisk().
Referenced by rollOffOldData().
|
inlineoverride |
Definition at line 238 of file FileMgr.h.
References logger::FATAL, and LOG.
|
inlineoverride |
void File_Namespace::FileMgr::closeRemovePhysical | ( | ) |
Definition at line 534 of file FileMgr.cpp.
References File_Namespace::close(), DBMetaFile_, epochFile_, files_, getFileMgrBasePath(), and File_Namespace::renameForDelete().
Referenced by File_Namespace::GlobalFileMgr::removeTableRelatedDS().
void File_Namespace::FileMgr::copyPage | ( | Page & | srcPage, |
FileMgr * | destFileMgr, | ||
Page & | destPage, | ||
const size_t | reservedHeaderSize, | ||
const size_t | numBytes, | ||
const size_t | offset | ||
) |
Definition at line 556 of file FileMgr.cpp.
References CHECK, checked_malloc(), defaultPageSize_, File_Namespace::Page::fileId, free(), getFileInfoForFileId(), File_Namespace::Page::pageNum, File_Namespace::FileInfo::read(), and File_Namespace::FileInfo::write().
Referenced by init().
bool File_Namespace::FileMgr::coreInit | ( | ) |
Determines file path, and if exists, runs file migration and opens and reads epoch file.
Definition at line 141 of file FileMgr.cpp.
References EPOCH_FILENAME, logger::FATAL, fileMgrBasePath_, fileMgrKey_, files_rw_mutex_, File_Namespace::GlobalFileMgr::getBasePath(), gfm_, LOG, migrateToLatestFileMgrVersion(), openAndReadEpochFile(), and to_string().
Referenced by FileMgr(), and init().
|
override |
Creates a chunk with the specified key and page size.
Definition at line 697 of file FileMgr.cpp.
References chunkIndexMutex_, and createBufferUnlocked().
|
private |
Definition at line 707 of file FileMgr.cpp.
References chunkIndex_, defaultPageSize_, logger::FATAL, LOG, and show_chunk().
Referenced by createBuffer(), and putBuffer().
|
private |
Definition at line 578 of file FileMgr.cpp.
References Epoch::byte_size(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, LOG, and writeAndSyncEpochToDisk().
Referenced by init(), and migrateEpochFileV0().
|
private |
Adds a file to the file manager repository.
This method will create a FileInfo object for the file being added, and it will create the corresponding file on physical disk with the indicated number of pages pre-allocated.
A pointer to the FileInfo object is returned, which itself has a file pointer (FILE*) and a file identifier (int32_t fileId).
fileName | The name given to the file in physical storage. |
pageSize | The logical page size for the pages in the file. |
numPages | The number of logical pages to initially allocate for the file. |
Definition at line 959 of file FileMgr.cpp.
References CHECK, File_Namespace::create(), generate_TableFunctionsFactory_init::f, logger::FATAL, fileIndex_, fileMgrBasePath_, files_, files_rw_mutex_, LOG, and nextFileId_.
Referenced by requestFreePage(), and requestFreePages().
void File_Namespace::FileMgr::createTopLevelMetadata | ( | ) |
Definition at line 1052 of file FileMgr.cpp.
References DB_META_FILENAME, db_version_, logger::FATAL, getDBVersion(), INVALID_VERSION, LOG, readVersionFromDisk(), and writeAndSyncVersionToDisk().
|
override |
Deletes the chunk with the specified key.
Definition at line 730 of file FileMgr.cpp.
References chunkIndex_, chunkIndexMutex_, logger::FATAL, LOG, and show_chunk().
|
override |
Definition at line 747 of file FileMgr.cpp.
References chunkIndex_, and chunkIndexMutex_.
|
inline |
Returns current value of epoch - should be one greater than recorded at last checkpoint.
Definition at line 245 of file FileMgr.h.
Referenced by File_Namespace::FileBuffer::append(), checkpoint(), File_Namespace::FileInfo::freePage(), File_Namespace::FileBuffer::freePagesBeforeEpoch(), init(), openAndReadLegacyEpochFile(), openExistingFile(), File_Namespace::FileBuffer::reserve(), and File_Namespace::FileBuffer::write().
|
inline |
Definition at line 247 of file FileMgr.h.
Referenced by getStorageStats().
|
override |
Definition at line 775 of file FileMgr.cpp.
References chunkIndex_, chunkIndexMutex_, Data_Namespace::AbstractBuffer::copyTo(), logger::FATAL, Data_Namespace::AbstractBuffer::isDirty(), LOG, show_chunk(), and Data_Namespace::AbstractBuffer::size().
|
override |
Definition at line 867 of file FileMgr.cpp.
References logger::FATAL, and LOG.
Referenced by copyPage().
void File_Namespace::FileMgr::free_page | ( | std::pair< FileInfo *, int32_t > && | page | ) |
Definition at line 1177 of file FileMgr.cpp.
References free_pages_, and mutex_free_page_.
Referenced by checkpoint(), File_Namespace::FileInfo::freePage(), and init().
|
private |
Definition at line 646 of file FileMgr.cpp.
References chunkIndex_.
Referenced by rollOffOldData().
|
inline |
Definition at line 308 of file FileMgr.h.
Referenced by File_Namespace::FileInfo::openExistingFile().
|
inlineoverride |
|
override |
Returns the a pointer to the chunk with the specified key.
Definition at line 767 of file FileMgr.cpp.
References CHECK, chunkIndex_, chunkIndexMutex_, and show_chunk().
|
override |
Definition at line 993 of file FileMgr.cpp.
References chunkIndex_, and chunkIndexMutex_.
bool File_Namespace::FileMgr::getDBConvert | ( | ) | const |
Definition at line 1048 of file FileMgr.cpp.
References File_Namespace::GlobalFileMgr::getDBConvert(), and gfm_.
int32_t File_Namespace::FileMgr::getDBVersion | ( | ) | const |
Index for looking up chunks.
Definition at line 1044 of file FileMgr.cpp.
References File_Namespace::GlobalFileMgr::getDBVersion(), and gfm_.
Referenced by createTopLevelMetadata().
FILE * File_Namespace::FileMgr::getFileForFileId | ( | const int32_t | fileId | ) |
Returns FILE pointer associated with requested fileId.
Definition at line 988 of file FileMgr.cpp.
Referenced by File_Namespace::FileBuffer::readMetadata(), and File_Namespace::FileBuffer::writeMetadata().
|
inline |
Definition at line 187 of file FileMgr.h.
Referenced by File_Namespace::FileBuffer::append(), File_Namespace::FileBuffer::copyPage(), copyPage(), File_Namespace::FileBuffer::freePage(), File_Namespace::readForThread(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeHeader().
|
inline |
Definition at line 302 of file FileMgr.h.
Referenced by closeRemovePhysical().
|
inlineoverride |
|
inlineoverride |
FileMetadata File_Namespace::FileMgr::getMetadataForFile | ( | const boost::filesystem::directory_iterator & | fileIterator | ) |
Definition at line 161 of file FileMgr.cpp.
References CHECK_EQ, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, omnisci::file_size(), File_Namespace::FileMetadata::file_size, File_Namespace::FileMetadata::is_data_file, LOG, MAPD_FILE_EXT, File_Namespace::FileMetadata::num_pages, and File_Namespace::FileMetadata::page_size.
Referenced by getStorageStats(), and init().
|
inlineoverride |
|
inlineoverride |
Definition at line 289 of file FileMgr.h.
|
inline |
Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.
Definition at line 278 of file FileMgr.h.
Referenced by File_Namespace::FileBuffer::read().
size_t File_Namespace::FileMgr::getNumUsedMetadataPages | ( | ) | const |
Definition at line 1025 of file FileMgr.cpp.
References chunkIndex_, and chunkIndexMutex_.
size_t File_Namespace::FileMgr::getNumUsedMetadataPagesForChunkKey | ( | const ChunkKey & | chunkKey | ) | const |
Definition at line 1034 of file FileMgr.cpp.
References chunkIndex_, and chunkIndexMutex_.
size_t File_Namespace::FileMgr::getNumUsedPages | ( | ) | const |
Definition at line 1016 of file FileMgr.cpp.
References files_, and files_rw_mutex_.
StorageStats File_Namespace::FileMgr::getStorageStats | ( | ) |
Definition at line 319 of file FileMgr.cpp.
References CHECK, File_Namespace::StorageStats::data_file_count, defaultPageSize_, File_Namespace::StorageStats::epoch, File_Namespace::StorageStats::epoch_floor, epochFloor(), logger::FATAL, File_Namespace::FileMetadata::file_path, File_Namespace::FileMetadata::file_size, fileMgrBasePath_, files_, files_rw_mutex_, getMetadataForFile(), File_Namespace::FileMetadata::is_data_file, isFullyInitted_, lastCheckpointedEpoch(), LOG, File_Namespace::StorageStats::metadata_file_count, METADATA_PAGE_SIZE, File_Namespace::FileMetadata::num_pages, File_Namespace::FileMetadata::page_size, File_Namespace::StorageStats::total_data_file_size, File_Namespace::StorageStats::total_data_page_count, File_Namespace::StorageStats::total_free_data_page_count, File_Namespace::StorageStats::total_free_metadata_page_count, File_Namespace::StorageStats::total_metadata_file_size, File_Namespace::StorageStats::total_metadata_page_count, and UNREACHABLE.
|
inlineoverride |
uint64_t File_Namespace::FileMgr::getTotalFileSize | ( | ) | const |
Definition at line 1186 of file FileMgr.cpp.
References DBMetaFile_, epochFile_, files_, and File_Namespace::fileSize().
Referenced by ShowDiskCacheUsageCommand::execute().
|
inline |
Definition at line 249 of file FileMgr.h.
References logger::FATAL, LOG, and Epoch::max_allowable_epoch().
Referenced by checkpoint(), and init().
void File_Namespace::FileMgr::init | ( | const size_t | num_reader_threads, |
const int32_t | epochOverride | ||
) |
Definition at line 196 of file FileMgr.cpp.
References Epoch::ceiling(), chunkIndex_, coreInit(), createEpochFile(), epoch(), epoch_, EPOCH_FILENAME, logger::FATAL, File_Namespace::FileMetadata::file_id, FILE_MGR_VERSION_FILENAME, File_Namespace::FileMetadata::file_path, fileMgrBasePath_, fileMgrVersion_, Epoch::floor(), free_page(), free_pages_, getMetadataForFile(), File_Namespace::headerCompare(), incrementEpoch(), logger::INFO, File_Namespace::FileMetadata::is_data_file, isFullyInitted_, latestFileMgrVersion_, LOG, mutex_free_page_, nextFileId_, File_Namespace::FileMetadata::num_pages, num_reader_threads_, openExistingFile(), File_Namespace::FileMetadata::page_size, processFileFutures(), rollOffOldData(), setEpoch(), gpu_enabled::sort(), timer_start(), timer_stop(), VLOG, writeAndSyncEpochToDisk(), and writeAndSyncVersionToDisk().
Referenced by FileMgr().
void File_Namespace::FileMgr::init | ( | const std::string & | dataPathToConvertFrom, |
const int32_t | epochOverride | ||
) |
Definition at line 405 of file FileMgr.cpp.
References CHECK, chunkIndex_, copyPage(), EPOCH_FILENAME, logger::FATAL, File_Namespace::FileMetadata::file_id, File_Namespace::FileMetadata::file_path, File_Namespace::GlobalFileMgr::getFileMgr(), getMetadataForFile(), File_Namespace::FileBuffer::getMultiPage(), gfm_, File_Namespace::headerCompare(), File_Namespace::FileMetadata::is_data_file, isFullyInitted_, LOG, File_Namespace::FileBuffer::multiPages_, nextFileId_, File_Namespace::FileMetadata::num_pages, openAndReadEpochFile(), openExistingFile(), File_Namespace::FileMetadata::page_size, File_Namespace::FileBuffer::pageDataSize(), File_Namespace::FileBuffer::pageSize(), processFileFutures(), File_Namespace::MultiPage::push(), requestFreePage(), File_Namespace::FileBuffer::reservedHeaderSize(), Data_Namespace::AbstractBuffer::setDirty(), setEpoch(), Data_Namespace::AbstractBuffer::setSize(), Data_Namespace::AbstractBuffer::size(), gpu_enabled::sort(), Data_Namespace::AbstractBuffer::syncEncoder(), and File_Namespace::FileBuffer::writeHeader().
|
inlineoverride |
|
override |
Definition at line 725 of file FileMgr.cpp.
References chunkIndex_, and chunkIndexMutex_.
|
inline |
Returns value of epoch at last checkpoint.
Definition at line 265 of file FileMgr.h.
Referenced by checkpoint(), File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr(), and getStorageStats().
|
inline |
Returns value max_rollback_epochs.
Definition at line 272 of file FileMgr.h.
Referenced by File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr().
|
private |
Definition at line 1114 of file FileMgr.cpp.
References Epoch::ceiling(), createEpochFile(), epoch_, EPOCH_FILENAME, FILE_MGR_VERSION_FILENAME, fileMgrBasePath_, Epoch::floor(), logger::INFO, LEGACY_EPOCH_FILENAME, LOG, Epoch::min_allowable_epoch(), openAndReadLegacyEpochFile(), writeAndSyncEpochToDisk(), and writeAndSyncVersionToDisk().
Referenced by migrateToLatestFileMgrVersion().
|
private |
Definition at line 1125 of file FileMgr.cpp.
References logger::FATAL, FILE_MGR_VERSION_FILENAME, fileMgrVersion_, INVALID_VERSION, latestFileMgrVersion_, LOG, migrateEpochFileV0(), readVersionFromDisk(), UNREACHABLE, and writeAndSyncVersionToDisk().
Referenced by coreInit().
|
private |
Definition at line 609 of file FileMgr.cpp.
References Epoch::byte_size(), epoch_, epochFile_, logger::FATAL, omnisci::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), File_Namespace::read(), and Epoch::storage_ptr().
Referenced by coreInit(), and init().
|
private |
Definition at line 589 of file FileMgr.cpp.
References epoch(), logger::FATAL, omnisci::file_size(), fileMgrBasePath_, LOG, File_Namespace::open(), and File_Namespace::read().
Referenced by migrateEpochFileV0().
|
private |
Definition at line 940 of file FileMgr.cpp.
References epoch(), generate_TableFunctionsFactory_init::f, fileIndex_, files_, files_rw_mutex_, File_Namespace::open(), and File_Namespace::FileInfo::openExistingFile().
Referenced by init().
|
inlineoverride |
|
private |
Definition at line 391 of file FileMgr.cpp.
Referenced by init().
|
override |
Puts the contents of d into the Chunk with the given key.
key | - Unique identifier for a Chunk. |
d | - An object representing the source data for the Chunk. |
Definition at line 804 of file FileMgr.cpp.
References File_Namespace::FileBuffer::append(), CHECK_LT, chunkIndex_, chunkIndexMutex_, Data_Namespace::AbstractBuffer::clearDirtyBits(), createBufferUnlocked(), defaultPageSize_, logger::FATAL, Data_Namespace::AbstractBuffer::getDeviceId(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::getType(), Data_Namespace::AbstractBuffer::isAppended(), Data_Namespace::AbstractBuffer::isDirty(), Data_Namespace::AbstractBuffer::isUpdated(), LOG, Data_Namespace::AbstractBuffer::setSize(), show_chunk(), Data_Namespace::AbstractBuffer::size(), Data_Namespace::AbstractBuffer::syncEncoder(), UNREACHABLE, and File_Namespace::FileBuffer::write().
|
private |
Definition at line 1068 of file FileMgr.cpp.
References File_Namespace::close(), omnisci::file_size(), fileMgrBasePath_, File_Namespace::open(), File_Namespace::read(), and setup::version.
Referenced by createTopLevelMetadata(), migrateToLatestFileMgrVersion(), and writeAndSyncVersionToDisk().
|
override |
Definition at line 1182 of file FileMgr.cpp.
References UNREACHABLE.
Page File_Namespace::FileMgr::requestFreePage | ( | size_t | pagesize, |
const bool | isMetadata | ||
) |
Definition at line 871 of file FileMgr.cpp.
References CHECK, createFile(), File_Namespace::FileInfo::fileId, fileIndex_, files_, File_Namespace::FileInfo::getFreePage(), getPageMutex_, MAX_FILE_N_METADATA_PAGES, and MAX_FILE_N_PAGES.
Referenced by File_Namespace::FileBuffer::addNewMultiPage(), init(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeMetadata().
void File_Namespace::FileMgr::requestFreePages | ( | size_t | npages, |
size_t | pagesize, | ||
std::vector< Page > & | pages, | ||
const bool | isMetadata | ||
) |
Obtains free pages – creates new files if necessary – of the requested size.
Given a page size and number of pages, this method updates the vector "pages" to include free pages of the requested size. These pages are immediately removed from the free list of the affected file(s). If there are not enough pages available among current files, new files are created and their pages are included in the vector.
npages | The number of free pages requested |
pagesize | The size of each requested page |
pages | A vector containing the free pages obtained by this method |
Definition at line 895 of file FileMgr.cpp.
References CHECK, createFile(), File_Namespace::FileInfo::fileId, fileIndex_, files_, File_Namespace::FileInfo::getFreePage(), getPageMutex_, MAX_FILE_N_METADATA_PAGES, and MAX_FILE_N_PAGES.
|
private |
Definition at line 652 of file FileMgr.cpp.
References checkpoint(), epoch_, Epoch::floor(), freePagesBeforeEpoch(), and maxRollbackEpochs_.
Referenced by checkpoint(), and init().
|
private |
Definition at line 1165 of file FileMgr.cpp.
References Epoch::ceiling(), epoch_, fileMgrKey_, Epoch::floor(), and writeAndSyncEpochToDisk().
Referenced by init().
|
private |
Definition at line 628 of file FileMgr.cpp.
References Epoch::byte_size(), CHECK, epoch_, epochFile_, epochIsCheckpointed_, logger::FATAL, omnisci::fsync(), LOG, Epoch::storage_ptr(), and File_Namespace::write().
Referenced by checkpoint(), createEpochFile(), init(), migrateEpochFileV0(), and setEpoch().
|
private |
Definition at line 1086 of file FileMgr.cpp.
References File_Namespace::close(), File_Namespace::create(), epochFile_, logger::FATAL, fileMgrBasePath_, omnisci::fsync(), logger::INFO, LOG, File_Namespace::open(), readVersionFromDisk(), and File_Namespace::write().
Referenced by createTopLevelMetadata(), init(), migrateEpochFileV0(), and migrateToLatestFileMgrVersion().
|
friend |
ChunkKeyToChunkMap File_Namespace::FileMgr::chunkIndex_ |
Definition at line 297 of file FileMgr.h.
Referenced by checkpoint(), createBufferUnlocked(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), freePagesBeforeEpoch(), getBuffer(), getChunkMetadataVecForKeyPrefix(), getNumUsedMetadataPages(), getNumUsedMetadataPagesForChunkKey(), init(), isBufferOnDevice(), putBuffer(), File_Namespace::GlobalFileMgr::writeFileMgrData(), and ~FileMgr().
|
mutableprivate |
Definition at line 339 of file FileMgr.h.
Referenced by checkpoint(), createBuffer(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), getBuffer(), getChunkMetadataVecForKeyPrefix(), getNumUsedMetadataPages(), getNumUsedMetadataPagesForChunkKey(), isBufferOnDevice(), and putBuffer().
|
private |
Definition at line 332 of file FileMgr.h.
Referenced by createTopLevelMetadata().
|
private |
Definition at line 336 of file FileMgr.h.
Referenced by closeRemovePhysical(), getTotalFileSize(), and ~FileMgr().
|
private |
number of threads used when loading data
Definition at line 325 of file FileMgr.h.
Referenced by copyPage(), createBufferUnlocked(), getStorageStats(), and putBuffer().
|
private |
the index of the next file id
Definition at line 327 of file FileMgr.h.
Referenced by FileMgr(), init(), migrateEpochFileV0(), openAndReadEpochFile(), rollOffOldData(), setEpoch(), and writeAndSyncEpochToDisk().
|
private |
Definition at line 331 of file FileMgr.h.
Referenced by closeRemovePhysical(), createEpochFile(), FileMgr(), getTotalFileSize(), openAndReadEpochFile(), writeAndSyncEpochToDisk(), writeAndSyncVersionToDisk(), and ~FileMgr().
|
private |
Definition at line 328 of file FileMgr.h.
Referenced by writeAndSyncEpochToDisk().
|
private |
A vector of files accessible via a file identifier.
Definition at line 323 of file FileMgr.h.
Referenced by createFile(), openExistingFile(), requestFreePage(), and requestFreePages().
|
private |
Definition at line 320 of file FileMgr.h.
Referenced by coreInit(), createEpochFile(), createFile(), FileMgr(), getStorageStats(), init(), migrateEpochFileV0(), openAndReadEpochFile(), openAndReadLegacyEpochFile(), readVersionFromDisk(), and writeAndSyncVersionToDisk().
|
private |
Global FileMgr.
Definition at line 318 of file FileMgr.h.
Referenced by checkpoint(), coreInit(), FileMgr(), and setEpoch().
|
private |
DB version from dbmeta file, should be compatible with GlobalFileMgr::omnisci_db_version_
Definition at line 334 of file FileMgr.h.
Referenced by init(), and migrateToLatestFileMgrVersion().
|
private |
The OS file system path containing files related to this FileMgr
Definition at line 322 of file FileMgr.h.
Referenced by checkpoint(), closeRemovePhysical(), createFile(), FileMgr(), getFileForFileId(), getNumUsedPages(), getStorageStats(), getTotalFileSize(), openExistingFile(), requestFreePage(), requestFreePages(), and ~FileMgr().
|
mutableprivate |
Definition at line 340 of file FileMgr.h.
Referenced by checkpoint(), coreInit(), createFile(), getNumUsedPages(), getStorageStats(), and openExistingFile().
|
private |
Definition at line 343 of file FileMgr.h.
Referenced by checkpoint(), free_page(), and init().
|
private |
pointer to DB level metadata
Definition at line 338 of file FileMgr.h.
Referenced by requestFreePage(), and requestFreePages().
|
private |
Definition at line 317 of file FileMgr.h.
Referenced by coreInit(), FileMgr(), getDBConvert(), getDBVersion(), and init().
|
private |
Definition at line 344 of file FileMgr.h.
Referenced by getStorageStats(), and init().
|
private |
Definition at line 335 of file FileMgr.h.
Referenced by init(), and migrateToLatestFileMgrVersion().
|
private |
Definition at line 319 of file FileMgr.h.
Referenced by rollOffOldData().
|
mutableprivate |
Definition at line 342 of file FileMgr.h.
Referenced by checkpoint(), free_page(), and init().
|
private |
Definition at line 326 of file FileMgr.h.
Referenced by createFile(), and init().
|
private |