25 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
26 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
32 #include "../Shared/heavyai_shared_mutex.h"
34 #include "../AbstractBuffer.h"
35 #include "../AbstractBufferMgr.h"
40 using namespace Data_Namespace;
42 namespace File_Namespace {
59 std::shared_ptr<ForeignStorageInterface>
fsi,
60 std::string basePath =
".",
61 const size_t num_reader_threads = 0,
69 const size_t numBytes = 0)
override {
70 return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
74 return getFileMgr(key)->isBufferOnDevice(key);
82 return getFileMgr(key)->deleteBuffer(key, purge);
85 void deleteBuffersWithPrefix(
const ChunkKey& keyPrefix,
86 const bool purge =
true)
override;
90 return getFileMgr(key)->getBuffer(key, numBytes);
95 const size_t numBytes)
override {
96 return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
107 const size_t numBytes = 0)
override {
108 return getFileMgr(key)->putBuffer(key, d, numBytes);
113 LOG(
FATAL) <<
"Operation not supported";
119 inline MgrType
getMgrType()
override {
return GLOBAL_FILE_MGR; };
121 inline std::string
printSlabs()
override {
return "Not Implemented"; }
130 const ChunkKey& keyPrefix)
override {
131 return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
139 void checkpoint()
override;
140 void checkpoint(
const int32_t db_id,
const int32_t tb_id)
override;
148 size_t getNumChunks()
override;
150 void compactDataFiles(
const int32_t db_id,
const int32_t tb_id);
153 AbstractBufferMgr* findFileMgrUnlocked(
const int32_t db_id,
const int32_t tb_id);
154 void deleteFileMgr(
const int32_t db_id,
const int32_t tb_id);
157 AbstractBufferMgr*
findFileMgr(
const int32_t db_id,
const int32_t tb_id) {
159 return findFileMgrUnlocked(db_id, tb_id);
161 void setFileMgrParams(
const int32_t db_id,
164 AbstractBufferMgr* getFileMgr(
const int32_t db_id,
const int32_t tb_id);
166 return getFileMgr(key[0], key[1]);
172 void writeFileMgrData(
FileMgr* fileMgr = 0);
178 void removeTableRelatedDS(
const int32_t db_id,
const int32_t tb_id)
override;
179 void setTableEpoch(
const int32_t db_id,
const int32_t tb_id,
const int32_t start_epoch);
180 size_t getTableEpoch(
const int32_t db_id,
const int32_t tb_id);
181 void resetTableEpochFloor(
const int32_t db_id,
const int32_t tb_id);
182 StorageStats getStorageStats(
const int32_t db_id,
const int32_t tb_id);
185 std::shared_ptr<FileMgr> getSharedFileMgr(
const int db_id,
const int table_id);
188 void setFileMgr(
const int db_id,
const int table_id, std::shared_ptr<FileMgr> file_mgr);
189 void closeFileMgr(
const int32_t db_id,
190 const int32_t tb_id);
193 std::shared_ptr<ForeignStorageInterface>
fsi_;
196 bool existsDiffBetweenFileMgrParamsAndFileMgr(
228 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
std::vector< int > ChunkKey
std::string getBasePath() const
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
heavyai::shared_lock< heavyai::shared_mutex > read_lock
bool isBufferOnDevice(const ChunkKey &key) override
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
void setDBConvert(bool val)
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
This file includes the class specification for the FILE manager (FileMgr), and related data structure...
bool getDBConvert() const
AbstractBufferMgr * getFileMgr(const ChunkKey &key)
int32_t getDBVersion() const
std::string getStringMgrType() override
std::shared_lock< T > shared_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
size_t getAllocated() override
void init(LogOptions const &log_opts)
~GlobalFileMgr() override
int32_t omnisci_db_version_
default page size, used to set FileMgr defaultPageSize_
An AbstractBuffer is a unit of data management for a data manager.
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::string printSlabs() override
#define DEFAULT_PAGE_SIZE
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
size_t getDefaultPageSize() const
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
void free(AbstractBuffer *buffer) override
size_t num_reader_threads_
The OS file system path containing the files.
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
std::map< TablePair, StorageStats > lazy_initialized_stats_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
std::shared_timed_mutex shared_mutex
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
int32_t max_rollback_epochs
size_t getInUseSize() override
heavyai::shared_mutex fileMgrs_mutex_