27 #include <boost/filesystem.hpp>
28 #include <boost/lexical_cast.hpp>
40 namespace File_Namespace {
42 GlobalFileMgr::GlobalFileMgr(
const int32_t device_id,
43 std::shared_ptr<ForeignStorageInterface> fsi,
44 std::string base_path,
45 const size_t num_reader_threads,
46 const size_t page_size,
47 const size_t metadata_page_size)
48 : AbstractBufferMgr(device_id)
50 , basePath_(base_path)
51 , num_reader_threads_(num_reader_threads)
54 , page_size_(page_size)
55 , metadata_page_size_(metadata_page_size) {
68 if (boost::filesystem::exists(path)) {
69 if (!boost::filesystem::is_directory(path)) {
70 LOG(
FATAL) <<
"Specified path is not a directory.";
73 if (!boost::filesystem::create_directory(path)) {
74 LOG(
FATAL) <<
"Could not create data directory";
83 fileMgrsIt->second->checkpoint();
93 size_t num_chunks = 0;
96 num_chunks += fileMgrsIt->second->getNumChunks();
108 if (keyPrefix[0] != -1) {
109 return getFileMgr(keyPrefix)->deleteBuffersWithPrefix(keyPrefix, purge);
114 const int32_t tb_id) {
116 AbstractBufferMgr* fm =
nullptr;
117 const auto file_mgr_key = std::make_pair(db_id, tb_id);
126 const auto file_mgr_key = std::make_pair(db_id, tb_id);
143 if (file_mgr_params.
epoch != -1 &&
162 const auto file_mgr_key = std::make_pair(db_id, tb_id);
163 auto max_rollback_epochs =
166 auto s = std::make_shared<FileMgr>(
195 const auto file_mgr_key = std::make_pair(db_id, tb_id);
196 const auto foreign_buffer_manager =
fsi_->lookupBufferManager(db_id, tb_id);
197 if (foreign_buffer_manager) {
200 return foreign_buffer_manager;
202 int32_t max_rollback_epochs{-1};
207 auto s = std::make_shared<FileMgr>(
219 const int table_id) {
220 const auto table_key = std::make_pair(db_id, table_id);
230 std::shared_ptr<FileMgr> file_mgr) {
244 if ((fileMgr != 0) && (fileMgr != fm)) {
249 chunkIt->second->write((int8_t*)chunkIt->second, chunkIt->second->size(), 0);
257 if (
auto fm = dynamic_cast<File_Namespace::FileMgr*>(abm)) {
258 fm->closeRemovePhysical();
259 }
else if (dynamic_cast<ForeignStorageBufferMgr*>(abm)) {
260 abm->removeTableRelatedDS(db_id, tb_id);
261 fsi_->dropBufferManager(db_id, tb_id);
266 const auto file_mgr_key = std::make_pair(db_id, tb_id);
267 auto u = std::make_unique<FileMgr>(0,
this, file_mgr_key,
true);
268 u->closeRemovePhysical();
278 const int32_t start_epoch) {
279 AbstractBufferMgr* opened_fm =
findFileMgr(db_id, tb_id);
285 const auto file_mgr_key = std::make_pair(db_id, tb_id);
289 auto u = std::make_unique<FileMgr>(
299 AbstractBufferMgr* opened_fm =
findFileMgr(db_id, tb_id);
301 return dynamic_cast<FileMgr*
>(opened_fm)->lastCheckpointedEpoch();
304 const auto file_mgr_key = std::make_pair(db_id, tb_id);
305 auto u = std::make_unique<FileMgr>(0,
this, file_mgr_key,
true);
306 const auto epoch = u->lastCheckpointedEpoch();
312 AbstractBufferMgr* fm =
getFileMgr(db_id, tb_id);
314 dynamic_cast<FileMgr*
>(fm)->resetEpochFloor();
319 AbstractBufferMgr* opened_fm =
findFileMgr(db_id, tb_id);
329 auto u = std::make_unique<FileMgr>(0,
this, file_mgr_key,
true);
332 return lazy_initialized_stats_[file_mgr_key];
341 file_mgr->compactFiles();
void writeFileMgrData(FileMgr *fileMgr=0)
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
std::vector< int > ChunkKey
void deleteFileMgr(const int32_t db_id, const int32_t tb_id)
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
heavyai::shared_lock< heavyai::shared_mutex > read_lock
This file includes the class specification for the FILE manager (FileMgr), and related data structure...
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
heavyai::unique_lock< heavyai::shared_mutex > write_lock
size_t getNumChunks() override
int32_t lastCheckpointedEpoch() const
Returns value of epoch at last checkpoint.
std::shared_lock< T > shared_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id)
void setTableEpoch(const int32_t db_id, const int32_t tb_id, const int32_t start_epoch)
StorageStats getStorageStats(const int32_t db_id, const int32_t tb_id)
AbstractBufferMgr * findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id)
ChunkKeyToChunkMap chunkIndex_
std::unique_lock< T > unique_lock
std::shared_ptr< FileMgr > getSharedFileMgr(const int db_id, const int table_id)
int32_t omnisci_db_version_
used to set FileMgr metadta_page_size_
bool existsDiffBetweenFileMgrParamsAndFileMgr(FileMgr *file_mgr, const FileMgrParams &file_mgr_params) const
void compactDataFiles(const int32_t db_id, const int32_t tb_id)
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
size_t num_reader_threads_
The OS file system path containing the files.
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
void setFileMgr(const int db_id, const int table_id, std::shared_ptr< FileMgr > file_mgr)
void closeFileMgr(const int32_t db_id, const int32_t tb_id)
void setFileMgrParams(const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params)
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
std::map< TablePair, StorageStats > lazy_initialized_stats_
int32_t maxRollbackEpochs()
Returns value max_rollback_epochs.
std::pair< const int32_t, const int32_t > TablePair
int32_t max_rollback_epochs
void removeTableRelatedDS(const int32_t db_id, const int32_t tb_id) override
heavyai::shared_mutex fileMgrs_mutex_
A selection of helper methods for File I/O.
size_t getTableEpoch(const int32_t db_id, const int32_t tb_id)