27 #include <boost/filesystem.hpp>
28 #include <boost/lexical_cast.hpp>
40 namespace File_Namespace {
42 GlobalFileMgr::GlobalFileMgr(
const int32_t deviceId,
43 std::shared_ptr<ForeignStorageInterface> fsi,
45 const size_t num_reader_threads,
46 const size_t defaultPageSize)
47 : AbstractBufferMgr(deviceId)
49 , num_reader_threads_(num_reader_threads)
53 defaultPageSize_(defaultPageSize)
67 if (boost::filesystem::exists(path)) {
68 if (!boost::filesystem::is_directory(path)) {
69 LOG(
FATAL) <<
"Specified path is not a directory.";
72 if (!boost::filesystem::create_directory(path)) {
73 LOG(
FATAL) <<
"Could not create data directory";
82 fileMgrsIt->second->checkpoint();
92 size_t num_chunks = 0;
95 num_chunks += fileMgrsIt->second->getNumChunks();
107 if (keyPrefix[0] != -1) {
108 return getFileMgr(keyPrefix)->deleteBuffersWithPrefix(keyPrefix, purge);
113 const int32_t tb_id) {
115 AbstractBufferMgr* fm =
nullptr;
116 const auto file_mgr_key = std::make_pair(db_id, tb_id);
125 const auto file_mgr_key = std::make_pair(db_id, tb_id);
142 if (file_mgr_params.
epoch != -1 &&
161 const auto file_mgr_key = std::make_pair(db_id, tb_id);
162 auto max_rollback_epochs =
165 auto s = std::make_shared<FileMgr>(
194 const auto file_mgr_key = std::make_pair(db_id, tb_id);
195 const auto foreign_buffer_manager =
fsi_->lookupBufferManager(db_id, tb_id);
196 if (foreign_buffer_manager) {
199 return foreign_buffer_manager;
201 int32_t max_rollback_epochs{-1};
206 auto s = std::make_shared<FileMgr>(0,
222 const int table_id) {
223 const auto table_key = std::make_pair(db_id, table_id);
233 std::shared_ptr<FileMgr> file_mgr) {
245 if ((fileMgr != 0) && (fileMgr != fm)) {
250 chunkIt->second->write((int8_t*)chunkIt->second, chunkIt->second->size(), 0);
264 const auto file_mgr_key = std::make_pair(db_id, tb_id);
265 auto u = std::make_unique<FileMgr>(0,
this, file_mgr_key,
defaultPageSize_,
true);
266 u->closeRemovePhysical();
276 const int32_t start_epoch) {
277 AbstractBufferMgr* opened_fm =
findFileMgr(db_id, tb_id);
283 const auto file_mgr_key = std::make_pair(db_id, tb_id);
287 auto u = std::make_unique<FileMgr>(
297 AbstractBufferMgr* opened_fm =
findFileMgr(db_id, tb_id);
299 return dynamic_cast<FileMgr*
>(opened_fm)->lastCheckpointedEpoch();
302 const auto file_mgr_key = std::make_pair(db_id, tb_id);
303 auto u = std::make_unique<FileMgr>(0,
this, file_mgr_key,
defaultPageSize_,
true);
304 const auto epoch = u->lastCheckpointedEpoch();
311 AbstractBufferMgr* opened_fm =
findFileMgr(db_id, tb_id);
316 const auto file_mgr_key = std::make_pair(db_id, tb_id);
317 auto u = std::make_unique<FileMgr>(0,
this, file_mgr_key,
defaultPageSize_,
true);
318 const auto basic_storage_stats = u->getStorageStats();
320 return basic_storage_stats;
328 file_mgr->compactFiles();
void writeFileMgrData(FileMgr *fileMgr=0)
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
std::vector< int > ChunkKey
void deleteFileMgr(const int32_t db_id, const int32_t tb_id)
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
std::map< FileMgrKey, std::shared_ptr< FileMgr > > ownedFileMgrs_
size_t getNumChunks() override
void setTableEpoch(const int32_t db_id, const int32_t tb_id, const int32_t start_epoch)
StorageStats getStorageStats(const int32_t db_id, const int32_t tb_id)
std::map< FileMgrKey, int32_t > max_rollback_epochs_per_table_
AbstractBufferMgr * findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id)
ChunkKeyToChunkMap chunkIndex_
std::shared_ptr< FileMgr > getSharedFileMgr(const int db_id, const int table_id)
int32_t omnisci_db_version_
default page size, used to set FileMgr defaultPageSize_
bool existsDiffBetweenFileMgrParamsAndFileMgr(FileMgr *file_mgr, const FileMgrParams &file_mgr_params) const
void compactDataFiles(const int32_t db_id, const int32_t tb_id)
virtual void closeRemovePhysical()
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
mapd_shared_lock< mapd_shared_mutex > read_lock
size_t num_reader_threads_
The OS file system path containing the files.
void setFileMgr(const int db_id, const int table_id, std::shared_ptr< FileMgr > file_mgr)
void closeFileMgr(const int32_t db_id, const int32_t tb_id)
void setFileMgrParams(const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params)
std::map< FileMgrKey, AbstractBufferMgr * > allFileMgrs_
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
mapd_shared_mutex fileMgrs_mutex_
mapd_unique_lock< mapd_shared_mutex > write_lock
int32_t maxRollbackEpochs()
Returns value max_rollback_epochs.
int32_t max_rollback_epochs
void removeTableRelatedDS(const int32_t db_id, const int32_t tb_id) override
A selection of helper methods for File I/O.
int32_t lastCheckpointedEpoch()
Returns value of epoch at last checkpoint.
size_t getTableEpoch(const int32_t db_id, const int32_t tb_id)