24 #include <boost/filesystem.hpp>
25 #include <boost/lexical_cast.hpp>
27 #include "../../Shared/File.h"
38 namespace File_Namespace {
40 GlobalFileMgr::GlobalFileMgr(
const int deviceId,
42 const size_t num_reader_threads,
43 const size_t defaultPageSize)
46 , num_reader_threads_(num_reader_threads)
50 defaultPageSize_(defaultPageSize) {
59 for (
auto fileMgrsIt =
fileMgrs_.begin(); fileMgrsIt !=
fileMgrs_.end(); ++fileMgrsIt) {
60 delete fileMgrsIt->second;
70 if (boost::filesystem::exists(path)) {
71 if (!boost::filesystem::is_directory(path)) {
72 LOG(
FATAL) <<
"Specified path is not a directory.";
75 if (!boost::filesystem::create_directory(path)) {
76 LOG(
FATAL) <<
"Could not create data directory";
83 for (
auto fileMgrsIt =
fileMgrs_.begin(); fileMgrsIt !=
fileMgrs_.end(); ++fileMgrsIt) {
84 fileMgrsIt->second->checkpoint();
95 size_t num_chunks = 0;
98 num_chunks += fileMgrsIt->second->getNumChunks();
111 if (keyPrefix[0] != -1) {
117 std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec) {
119 std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVecForFileMgr;
120 for (
auto fileMgrsIt =
fileMgrs_.begin(); fileMgrsIt !=
fileMgrs_.end(); ++fileMgrsIt) {
121 fileMgrsIt->second->getChunkMetadataVec(chunkMetadataVecForFileMgr);
122 while (!chunkMetadataVecForFileMgr.empty()) {
124 chunkMetadataVec.push_back(chunkMetadataVecForFileMgr.back());
125 chunkMetadataVecForFileMgr.pop_back();
132 const bool removeFromMap) {
134 const auto file_mgr_key = std::make_pair(db_id, tb_id);
157 const auto file_mgr_key = std::make_pair(db_id, tb_id);
165 auto it_ok =
fileMgrs_.insert(std::make_pair(file_mgr_key, fm));
175 FileMgr* fm = fileMgrIt->second;
176 if ((fileMgr != 0) && (fileMgr != fm)) {
181 chunkIt->second->write((int8_t*)chunkIt->second, chunkIt->second->size(), 0);
194 const auto file_mgr_key = std::make_pair(db_id, tb_id);
195 fm =
new FileMgr(0,
this, file_mgr_key,
true);
205 const int start_epoch) {
206 const auto file_mgr_key = std::make_pair(db_id, tb_id);
219 LOG(
INFO) <<
"found and removed fm";
void writeFileMgrData(FileMgr *fileMgr=0)
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
std::vector< int > ChunkKey
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
int epoch_
number of threads used when loading data
FileMgr * getFileMgr(const int db_id, const int tb_id)
void getChunkMetadataVec(std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec) override
size_t getTableEpoch(const int db_id, const int tb_id)
size_t getNumChunks() override
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
FileMgr * findFileMgr(const int db_id, const int tb_id, const bool removeFromMap=false)
void setTableEpoch(const int db_id, const int tb_id, const int start_epoch)
int mapd_db_version_
default page size, used to set FileMgr defaultPageSize_
int epoch_
the index of the next file id
ChunkKeyToChunkMap chunkIndex_
void removeTableRelatedDS(const int db_id, const int tb_id)
~GlobalFileMgr() override
Destructor.
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
std::map< std::pair< int, int >, FileMgr * > fileMgrs_
void closeRemovePhysical()
size_t num_reader_threads_
The OS file system path containing the files.
mapd_shared_mutex fileMgrs_mutex_