OmniSciDB
cde582ebc3
|
#include <GlobalFileMgr.h>
Public Member Functions | |
GlobalFileMgr (const int32_t deviceId, std::shared_ptr< ForeignStorageInterface > fsi, std::string basePath=".", const size_t num_reader_threads=0, const size_t defaultPageSize=DEFAULT_PAGE_SIZE) | |
Constructor. More... | |
~GlobalFileMgr () override | |
AbstractBuffer * | createBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override |
Creates a chunk with the specified key and page size. More... | |
bool | isBufferOnDevice (const ChunkKey &key) override |
void | deleteBuffer (const ChunkKey &key, const bool purge=true) override |
Deletes the chunk with the specified key. More... | |
void | deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override |
AbstractBuffer * | getBuffer (const ChunkKey &key, const size_t numBytes=0) override |
Returns the a pointer to the chunk with the specified key. More... | |
void | fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override |
AbstractBuffer * | putBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override |
Puts the contents of d into the Chunk with the given key. More... | |
AbstractBuffer * | alloc (const size_t numBytes) override |
void | free (AbstractBuffer *buffer) override |
MgrType | getMgrType () override |
std::string | getStringMgrType () override |
std::string | printSlabs () override |
size_t | getMaxSize () override |
size_t | getInUseSize () override |
size_t | getAllocated () override |
bool | isAllocationCapped () override |
void | init () |
void | getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override |
void | checkpoint () override |
Fsyncs data files, writes out epoch and fsyncs that. More... | |
void | checkpoint (const int32_t db_id, const int32_t tb_id) override |
size_t | getNumReaderThreads () |
Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More... | |
size_t | getNumChunks () override |
void | compactDataFiles (const int32_t db_id, const int32_t tb_id) |
AbstractBufferMgr * | findFileMgr (const int32_t db_id, const int32_t tb_id) |
void | setFileMgrParams (const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params) |
AbstractBufferMgr * | getFileMgr (const int32_t db_id, const int32_t tb_id) |
AbstractBufferMgr * | getFileMgr (const ChunkKey &key) |
std::string | getBasePath () const |
size_t | getDefaultPageSize () const |
void | writeFileMgrData (FileMgr *fileMgr=0) |
int32_t | getDBVersion () const |
bool | getDBConvert () const |
void | setDBConvert (bool val) |
void | removeTableRelatedDS (const int32_t db_id, const int32_t tb_id) override |
void | setTableEpoch (const int32_t db_id, const int32_t tb_id, const int32_t start_epoch) |
size_t | getTableEpoch (const int32_t db_id, const int32_t tb_id) |
void | resetTableEpochFloor (const int32_t db_id, const int32_t tb_id) |
StorageStats | getStorageStats (const int32_t db_id, const int32_t tb_id) |
std::shared_ptr< FileMgr > | getSharedFileMgr (const int db_id, const int table_id) |
void | setFileMgr (const int db_id, const int table_id, std::shared_ptr< FileMgr > file_mgr) |
void | closeFileMgr (const int32_t db_id, const int32_t tb_id) |
Protected Attributes | |
std::shared_ptr < ForeignStorageInterface > | fsi_ |
Private Member Functions | |
AbstractBufferMgr * | findFileMgrUnlocked (const int32_t db_id, const int32_t tb_id) |
void | deleteFileMgr (const int32_t db_id, const int32_t tb_id) |
bool | existsDiffBetweenFileMgrParamsAndFileMgr (FileMgr *file_mgr, const FileMgrParams &file_mgr_params) const |
Private Attributes | |
std::string | basePath_ |
size_t | num_reader_threads_ |
The OS file system path containing the files. More... | |
int32_t | epoch_ |
number of threads used when loading data More... | |
size_t | defaultPageSize_ |
int32_t | omnisci_db_version_ |
default page size, used to set FileMgr defaultPageSize_ More... | |
bool | dbConvert_ |
std::map< TablePair, std::shared_ptr< FileMgr > > | ownedFileMgrs_ |
std::map< TablePair, AbstractBufferMgr * > | allFileMgrs_ |
std::map< TablePair, int32_t > | max_rollback_epochs_per_table_ |
std::map< TablePair, StorageStats > | lazy_initialized_stats_ |
heavyai::shared_mutex | fileMgrs_mutex_ |
Definition at line 54 of file GlobalFileMgr.h.
File_Namespace::GlobalFileMgr::GlobalFileMgr | ( | const int32_t | deviceId, |
std::shared_ptr< ForeignStorageInterface > | fsi, | ||
std::string | basePath = "." , |
||
const size_t | num_reader_threads = 0 , |
||
const size_t | defaultPageSize = DEFAULT_PAGE_SIZE |
||
) |
Constructor.
Definition at line 42 of file GlobalFileMgr.cpp.
References dbConvert_, init(), and omnisci_db_version_.
|
inlineoverride |
Definition at line 64 of file GlobalFileMgr.h.
|
inlineoverride |
Definition at line 112 of file GlobalFileMgr.h.
References logger::FATAL, and LOG.
|
override |
Fsyncs data files, writes out epoch and fsyncs that.
Definition at line 78 of file GlobalFileMgr.cpp.
References allFileMgrs_, and fileMgrs_mutex_.
Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint().
|
override |
Definition at line 86 of file GlobalFileMgr.cpp.
References getFileMgr().
void File_Namespace::GlobalFileMgr::closeFileMgr | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 134 of file GlobalFileMgr.cpp.
References deleteFileMgr(), and fileMgrs_mutex_.
void File_Namespace::GlobalFileMgr::compactDataFiles | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 341 of file GlobalFileMgr.cpp.
References deleteFileMgr(), fileMgrs_mutex_, findFileMgr(), and getFileMgr().
Referenced by TableOptimizer::vacuumDeletedRows().
|
inlineoverride |
Creates a chunk with the specified key and page size.
Definition at line 67 of file GlobalFileMgr.h.
Referenced by File_Namespace::CachingGlobalFileMgr::createBuffer().
|
inlineoverride |
Deletes the chunk with the specified key.
Definition at line 81 of file GlobalFileMgr.h.
Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffer().
|
override |
Definition at line 101 of file GlobalFileMgr.cpp.
References getFileMgr().
Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffersWithPrefix().
|
private |
Definition at line 123 of file GlobalFileMgr.cpp.
References allFileMgrs_, and ownedFileMgrs_.
Referenced by closeFileMgr(), compactDataFiles(), removeTableRelatedDS(), setFileMgrParams(), and setTableEpoch().
|
private |
Definition at line 139 of file GlobalFileMgr.cpp.
References File_Namespace::FileMgrParams::epoch, File_Namespace::FileMgr::lastCheckpointedEpoch(), File_Namespace::FileMgrParams::max_rollback_epochs, and File_Namespace::FileMgr::maxRollbackEpochs().
|
inlineoverride |
Definition at line 93 of file GlobalFileMgr.h.
Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint(), and File_Namespace::CachingGlobalFileMgr::fetchBuffer().
|
inline |
Definition at line 157 of file GlobalFileMgr.h.
Referenced by compactDataFiles(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), getStorageStats(), getTableEpoch(), setFileMgrParams(), and setTableEpoch().
|
private |
Definition at line 112 of file GlobalFileMgr.cpp.
References allFileMgrs_.
Referenced by getFileMgr(), and removeTableRelatedDS().
|
inlineoverride |
Definition at line 117 of file GlobalFileMgr.h.
References logger::FATAL, and LOG.
|
inlineoverride |
Definition at line 124 of file GlobalFileMgr.h.
|
inline |
Definition at line 169 of file GlobalFileMgr.h.
Referenced by anonymous_namespace{TableArchiver.cpp}::abs_path(), File_Namespace::FileMgr::coreInit(), and File_Namespace::FileMgr::FileMgr().
|
inlineoverride |
Returns the a pointer to the chunk with the specified key.
Definition at line 89 of file GlobalFileMgr.h.
Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint().
|
inlineoverride |
Definition at line 129 of file GlobalFileMgr.h.
Referenced by File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix().
|
inline |
Definition at line 175 of file GlobalFileMgr.h.
Referenced by File_Namespace::FileMgr::getDBConvert().
|
inline |
Definition at line 174 of file GlobalFileMgr.h.
Referenced by File_Namespace::FileMgr::getDBVersion().
|
inline |
Definition at line 170 of file GlobalFileMgr.h.
AbstractBufferMgr * File_Namespace::GlobalFileMgr::getFileMgr | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 180 of file GlobalFileMgr.cpp.
References allFileMgrs_, CHECK, defaultPageSize_, epoch_, fileMgrs_mutex_, findFileMgrUnlocked(), fsi_, lazy_initialized_stats_, max_rollback_epochs_per_table_, num_reader_threads_, and ownedFileMgrs_.
Referenced by checkpoint(), compactDataFiles(), deleteBuffersWithPrefix(), File_Namespace::FileMgr::init(), and resetTableEpochFloor().
|
inline |
Definition at line 165 of file GlobalFileMgr.h.
|
inlineoverride |
Definition at line 123 of file GlobalFileMgr.h.
|
inlineoverride |
Definition at line 122 of file GlobalFileMgr.h.
|
inlineoverride |
Definition at line 119 of file GlobalFileMgr.h.
|
override |
Definition at line 90 of file GlobalFileMgr.cpp.
References allFileMgrs_, and fileMgrs_mutex_.
|
inline |
Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.
Definition at line 146 of file GlobalFileMgr.h.
std::shared_ptr< FileMgr > File_Namespace::GlobalFileMgr::getSharedFileMgr | ( | const int | db_id, |
const int | table_id | ||
) |
Definition at line 223 of file GlobalFileMgr.cpp.
References ownedFileMgrs_.
StorageStats File_Namespace::GlobalFileMgr::getStorageStats | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 322 of file GlobalFileMgr.cpp.
References defaultPageSize_, fileMgrs_mutex_, findFileMgr(), and lazy_initialized_stats_.
|
inlineoverride |
Definition at line 120 of file GlobalFileMgr.h.
size_t File_Namespace::GlobalFileMgr::getTableEpoch | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 300 of file GlobalFileMgr.cpp.
References defaultPageSize_, fileMgrs_mutex_, and findFileMgr().
void File_Namespace::GlobalFileMgr::init | ( | ) |
Definition at line 61 of file GlobalFileMgr.cpp.
References basePath_, logger::FATAL, and LOG.
Referenced by GlobalFileMgr().
|
inlineoverride |
Definition at line 125 of file GlobalFileMgr.h.
|
inlineoverride |
Definition at line 73 of file GlobalFileMgr.h.
|
inlineoverride |
Definition at line 121 of file GlobalFileMgr.h.
|
inlineoverride |
Puts the contents of d into the Chunk with the given key.
key | - Unique identifier for a Chunk. |
d | - An object representing the source data for the Chunk. |
Definition at line 105 of file GlobalFileMgr.h.
Referenced by File_Namespace::CachingGlobalFileMgr::putBuffer().
|
override |
Definition at line 259 of file GlobalFileMgr.cpp.
References defaultPageSize_, deleteFileMgr(), fileMgrs_mutex_, findFileMgrUnlocked(), fsi_, and max_rollback_epochs_per_table_.
Referenced by File_Namespace::CachingGlobalFileMgr::removeTableRelatedDS().
void File_Namespace::GlobalFileMgr::resetTableEpochFloor | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 316 of file GlobalFileMgr.cpp.
References CHECK, and getFileMgr().
|
inline |
Definition at line 176 of file GlobalFileMgr.h.
void File_Namespace::GlobalFileMgr::setFileMgr | ( | const int | db_id, |
const int | table_id, | ||
std::shared_ptr< FileMgr > | file_mgr | ||
) |
Definition at line 233 of file GlobalFileMgr.cpp.
References allFileMgrs_, lazy_initialized_stats_, and ownedFileMgrs_.
void File_Namespace::GlobalFileMgr::setFileMgrParams | ( | const int32_t | db_id, |
const int32_t | tb_id, | ||
const FileMgrParams & | file_mgr_params | ||
) |
Definition at line 153 of file GlobalFileMgr.cpp.
References allFileMgrs_, CHECK, defaultPageSize_, deleteFileMgr(), File_Namespace::FileMgrParams::epoch, epoch_, fileMgrs_mutex_, findFileMgr(), lazy_initialized_stats_, File_Namespace::FileMgrParams::max_rollback_epochs, max_rollback_epochs_per_table_, num_reader_threads_, and ownedFileMgrs_.
void File_Namespace::GlobalFileMgr::setTableEpoch | ( | const int32_t | db_id, |
const int32_t | tb_id, | ||
const int32_t | start_epoch | ||
) |
Definition at line 281 of file GlobalFileMgr.cpp.
References defaultPageSize_, deleteFileMgr(), findFileMgr(), and num_reader_threads_.
void File_Namespace::GlobalFileMgr::writeFileMgrData | ( | FileMgr * | fileMgr = 0 | ) |
Definition at line 242 of file GlobalFileMgr.cpp.
References allFileMgrs_, CHECK, File_Namespace::FileMgr::chunkIndex_, and fileMgrs_mutex_.
|
private |
Definition at line 219 of file GlobalFileMgr.h.
Referenced by checkpoint(), deleteFileMgr(), findFileMgrUnlocked(), getFileMgr(), getNumChunks(), setFileMgr(), setFileMgrParams(), and writeFileMgrData().
|
private |
Definition at line 199 of file GlobalFileMgr.h.
Referenced by init().
|
private |
DB version for DataMgr DS and corresponding file buffer read/write code
Definition at line 215 of file GlobalFileMgr.h.
Referenced by GlobalFileMgr().
|
private |
Definition at line 206 of file GlobalFileMgr.h.
Referenced by getFileMgr(), getStorageStats(), getTableEpoch(), removeTableRelatedDS(), setFileMgrParams(), and setTableEpoch().
|
private |
number of threads used when loading data
Definition at line 202 of file GlobalFileMgr.h.
Referenced by getFileMgr(), and setFileMgrParams().
|
private |
Definition at line 223 of file GlobalFileMgr.h.
Referenced by checkpoint(), closeFileMgr(), compactDataFiles(), getFileMgr(), getNumChunks(), getStorageStats(), getTableEpoch(), removeTableRelatedDS(), setFileMgrParams(), and writeFileMgrData().
|
protected |
Definition at line 193 of file GlobalFileMgr.h.
Referenced by getFileMgr(), File_Namespace::CachingGlobalFileMgr::isChunkPrefixCacheable(), and removeTableRelatedDS().
|
private |
Definition at line 221 of file GlobalFileMgr.h.
Referenced by getFileMgr(), getStorageStats(), setFileMgr(), and setFileMgrParams().
|
private |
Definition at line 220 of file GlobalFileMgr.h.
Referenced by getFileMgr(), removeTableRelatedDS(), and setFileMgrParams().
|
private |
The OS file system path containing the files.
Definition at line 200 of file GlobalFileMgr.h.
Referenced by getFileMgr(), setFileMgrParams(), and setTableEpoch().
|
private |
default page size, used to set FileMgr defaultPageSize_
Definition at line 209 of file GlobalFileMgr.h.
Referenced by GlobalFileMgr().
|
private |
true if conversion should be done between different "omnisci_db_version_"
Definition at line 218 of file GlobalFileMgr.h.
Referenced by deleteFileMgr(), getFileMgr(), getSharedFileMgr(), setFileMgr(), and setFileMgrParams().