OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
File_Namespace::GlobalFileMgr Class Reference

#include <GlobalFileMgr.h>

+ Inheritance diagram for File_Namespace::GlobalFileMgr:
+ Collaboration diagram for File_Namespace::GlobalFileMgr:

Public Member Functions

 GlobalFileMgr (const int32_t device_id, std::shared_ptr< ForeignStorageInterface > fsi, std::string base_path=".", const size_t num_reader_threads=0, const size_t page_size=DEFAULT_PAGE_SIZE, const size_t metadata_page_size=DEFAULT_METADATA_PAGE_SIZE)
 Constructor. More...
 
 ~GlobalFileMgr () override
 
AbstractBuffercreateBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
 Creates a chunk with the specified key and page size. More...
 
bool isBufferOnDevice (const ChunkKey &key) override
 
void deleteBuffer (const ChunkKey &key, const bool purge=true) override
 Deletes the chunk with the specified key. More...
 
void deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
 
AbstractBuffergetBuffer (const ChunkKey &key, const size_t numBytes=0) override
 Returns the a pointer to the chunk with the specified key. More...
 
void fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
 
AbstractBufferputBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
 Puts the contents of d into the Chunk with the given key. More...
 
AbstractBufferalloc (const size_t numBytes) override
 
void free (AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 
std::string getStringMgrType () override
 
std::string printSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void init ()
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
 
void checkpoint () override
 Fsyncs data files, writes out epoch and fsyncs that. More...
 
void checkpoint (const int32_t db_id, const int32_t tb_id) override
 
size_t getNumReaderThreads ()
 Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...
 
size_t getNumChunks () override
 
void compactDataFiles (const int32_t db_id, const int32_t tb_id)
 
AbstractBufferMgrfindFileMgr (const int32_t db_id, const int32_t tb_id)
 
void setFileMgrParams (const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params)
 
AbstractBufferMgrgetFileMgr (const int32_t db_id, const int32_t tb_id)
 
AbstractBufferMgrgetFileMgr (const ChunkKey &key)
 
std::string getBasePath () const
 
size_t getPageSize () const
 
size_t getMetadataPageSize () const
 
void writeFileMgrData (FileMgr *fileMgr=0)
 
int32_t getDBVersion () const
 
bool getDBConvert () const
 
void setDBConvert (bool val)
 
void removeTableRelatedDS (const int32_t db_id, const int32_t tb_id) override
 
void setTableEpoch (const int32_t db_id, const int32_t tb_id, const int32_t start_epoch)
 
size_t getTableEpoch (const int32_t db_id, const int32_t tb_id)
 
void resetTableEpochFloor (const int32_t db_id, const int32_t tb_id)
 
StorageStats getStorageStats (const int32_t db_id, const int32_t tb_id)
 
std::shared_ptr< FileMgrgetSharedFileMgr (const int db_id, const int table_id)
 
void setFileMgr (const int db_id, const int table_id, std::shared_ptr< FileMgr > file_mgr)
 
void closeFileMgr (const int32_t db_id, const int32_t tb_id)
 

Protected Attributes

std::shared_ptr
< ForeignStorageInterface
fsi_
 

Private Member Functions

AbstractBufferMgrfindFileMgrUnlocked (const int32_t db_id, const int32_t tb_id)
 
void deleteFileMgr (const int32_t db_id, const int32_t tb_id)
 
bool existsDiffBetweenFileMgrParamsAndFileMgr (FileMgr *file_mgr, const FileMgrParams &file_mgr_params) const
 

Private Attributes

std::string basePath_
 
size_t num_reader_threads_
 The OS file system path containing the files. More...
 
int32_t epoch_
 number of threads used when loading data More...
 
const size_t page_size_
 
const size_t metadata_page_size_
 used to set FileMgr page_size_ More...
 
int32_t omnisci_db_version_
 used to set FileMgr metadta_page_size_ More...
 
bool dbConvert_
 
std::map< TablePair,
std::shared_ptr< FileMgr > > 
ownedFileMgrs_
 
std::map< TablePair,
AbstractBufferMgr * > 
allFileMgrs_
 
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
 
std::map< TablePair, StorageStatslazy_initialized_stats_
 
heavyai::shared_mutex fileMgrs_mutex_
 

Detailed Description

Definition at line 54 of file GlobalFileMgr.h.

Constructor & Destructor Documentation

File_Namespace::GlobalFileMgr::GlobalFileMgr ( const int32_t  device_id,
std::shared_ptr< ForeignStorageInterface fsi,
std::string  base_path = ".",
const size_t  num_reader_threads = 0,
const size_t  page_size = DEFAULT_PAGE_SIZE,
const size_t  metadata_page_size = DEFAULT_METADATA_PAGE_SIZE 
)

Constructor.

Definition at line 42 of file GlobalFileMgr.cpp.

References dbConvert_, init(), and omnisci_db_version_.

48  : AbstractBufferMgr(device_id)
49  , fsi_(fsi)
50  , basePath_(base_path)
51  , num_reader_threads_(num_reader_threads)
52  , epoch_(-1) // set the default epoch for all tables corresponding to the time of
53  // last checkpoint
54  , page_size_(page_size)
55  , metadata_page_size_(metadata_page_size) {
57  // DS changes also triggered by individual FileMgr per table project (release 2.1.0)
58  dbConvert_ = false;
59  init();
60 }
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
const size_t metadata_page_size_
used to set FileMgr page_size_
int32_t omnisci_db_version_
used to set FileMgr metadta_page_size_
size_t num_reader_threads_
The OS file system path containing the files.

+ Here is the call graph for this function:

File_Namespace::GlobalFileMgr::~GlobalFileMgr ( )
inlineoverride

Definition at line 65 of file GlobalFileMgr.h.

65 {}

Member Function Documentation

AbstractBuffer* File_Namespace::GlobalFileMgr::alloc ( const size_t  numBytes)
inlineoverride

Definition at line 113 of file GlobalFileMgr.h.

References logger::FATAL, and LOG.

113  {
114  LOG(FATAL) << "Operation not supported";
115  return nullptr; // satisfy return-type warning
116  }
#define LOG(tag)
Definition: Logger.h:285
void File_Namespace::GlobalFileMgr::checkpoint ( )
override

Fsyncs data files, writes out epoch and fsyncs that.

Definition at line 79 of file GlobalFileMgr.cpp.

References allFileMgrs_, and fileMgrs_mutex_.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint().

79  {
81  for (auto fileMgrsIt = allFileMgrs_.begin(); fileMgrsIt != allFileMgrs_.end();
82  ++fileMgrsIt) {
83  fileMgrsIt->second->checkpoint();
84  }
85 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
std::unique_lock< T > unique_lock
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::checkpoint ( const int32_t  db_id,
const int32_t  tb_id 
)
override

Definition at line 87 of file GlobalFileMgr.cpp.

References getFileMgr().

87  {
88  getFileMgr(db_id, tb_id)->checkpoint();
89 }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::closeFileMgr ( const int32_t  db_id,
const int32_t  tb_id 
)

Definition at line 135 of file GlobalFileMgr.cpp.

References deleteFileMgr(), and fileMgrs_mutex_.

135  {
137  deleteFileMgr(db_id, tb_id);
138 }
void deleteFileMgr(const int32_t db_id, const int32_t tb_id)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::unique_lock< T > unique_lock
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::compactDataFiles ( const int32_t  db_id,
const int32_t  tb_id 
)

Definition at line 336 of file GlobalFileMgr.cpp.

References deleteFileMgr(), fileMgrs_mutex_, findFileMgr(), and getFileMgr().

Referenced by TableOptimizer::vacuumDeletedRows().

336  {
337  auto file_mgr = dynamic_cast<File_Namespace::FileMgr*>(findFileMgr(db_id, tb_id));
338  {
340  if (file_mgr) {
341  file_mgr->compactFiles();
342  deleteFileMgr(db_id, tb_id);
343  }
344  }
345 
346  // Re-initialize file manager
347  getFileMgr(db_id, tb_id);
348 }
void deleteFileMgr(const int32_t db_id, const int32_t tb_id)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::unique_lock< T > unique_lock
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer* File_Namespace::GlobalFileMgr::createBuffer ( const ChunkKey key,
size_t  pageSize = 0,
const size_t  numBytes = 0 
)
inlineoverride

Creates a chunk with the specified key and page size.

Definition at line 68 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::createBuffer().

70  {
71  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
72  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::deleteBuffer ( const ChunkKey key,
const bool  purge = true 
)
inlineoverride

Deletes the chunk with the specified key.

Definition at line 82 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffer().

82  {
83  return getFileMgr(key)->deleteBuffer(key, purge);
84  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::deleteBuffersWithPrefix ( const ChunkKey keyPrefix,
const bool  purge = true 
)
override

Definition at line 102 of file GlobalFileMgr.cpp.

References getFileMgr().

Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffersWithPrefix().

102  {
103  /* keyPrefix[0] can be -1 only for gpu or cpu buffers but not for FileMgr.
104  * There is no assert here, as GlobalFileMgr is being called with -1 value as well in
105  * the same loop with other buffers. So the case of -1 will just be ignored, as nothing
106  * needs to be done.
107  */
108  if (keyPrefix[0] != -1) {
109  return getFileMgr(keyPrefix)->deleteBuffersWithPrefix(keyPrefix, purge);
110  }
111 }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::deleteFileMgr ( const int32_t  db_id,
const int32_t  tb_id 
)
private

Definition at line 124 of file GlobalFileMgr.cpp.

References allFileMgrs_, and ownedFileMgrs_.

Referenced by closeFileMgr(), compactDataFiles(), removeTableRelatedDS(), setFileMgrParams(), and setTableEpoch().

124  {
125  // NOTE: only call this private function after locking is already in place
126  const auto file_mgr_key = std::make_pair(db_id, tb_id);
127  if (auto it = ownedFileMgrs_.find(file_mgr_key); it != ownedFileMgrs_.end()) {
128  ownedFileMgrs_.erase(it);
129  }
130  if (auto it = allFileMgrs_.find(file_mgr_key); it != allFileMgrs_.end()) {
131  allFileMgrs_.erase(it);
132  }
133 }
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_

+ Here is the caller graph for this function:

bool File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr ( FileMgr file_mgr,
const FileMgrParams file_mgr_params 
) const
private

Definition at line 140 of file GlobalFileMgr.cpp.

References File_Namespace::FileMgrParams::epoch, File_Namespace::FileMgr::lastCheckpointedEpoch(), File_Namespace::FileMgrParams::max_rollback_epochs, and File_Namespace::FileMgr::maxRollbackEpochs().

142  {
143  if (file_mgr_params.epoch != -1 &&
144  file_mgr_params.epoch != file_mgr->lastCheckpointedEpoch()) {
145  return true;
146  }
147  if (file_mgr_params.max_rollback_epochs != -1 &&
148  file_mgr_params.max_rollback_epochs != file_mgr->maxRollbackEpochs()) {
149  return true;
150  }
151  return false;
152 }

+ Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::fetchBuffer ( const ChunkKey key,
AbstractBuffer destBuffer,
const size_t  numBytes 
)
inlineoverride

Definition at line 94 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint(), and File_Namespace::CachingGlobalFileMgr::fetchBuffer().

96  {
97  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
98  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the caller graph for this function:

AbstractBufferMgr* File_Namespace::GlobalFileMgr::findFileMgr ( const int32_t  db_id,
const int32_t  tb_id 
)
inline

Definition at line 158 of file GlobalFileMgr.h.

Referenced by compactDataFiles(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), getStorageStats(), getTableEpoch(), setFileMgrParams(), and setTableEpoch().

158  {
160  return findFileMgrUnlocked(db_id, tb_id);
161  }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::shared_lock< T > shared_lock
AbstractBufferMgr * findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id)
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the caller graph for this function:

AbstractBufferMgr * File_Namespace::GlobalFileMgr::findFileMgrUnlocked ( const int32_t  db_id,
const int32_t  tb_id 
)
private

Definition at line 113 of file GlobalFileMgr.cpp.

References allFileMgrs_.

Referenced by getFileMgr(), and removeTableRelatedDS().

114  {
115  // NOTE: only call this private function after locking is already in place
116  AbstractBufferMgr* fm = nullptr;
117  const auto file_mgr_key = std::make_pair(db_id, tb_id);
118  if (auto it = allFileMgrs_.find(file_mgr_key); it != allFileMgrs_.end()) {
119  fm = it->second;
120  }
121  return fm;
122 }
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::free ( AbstractBuffer buffer)
inlineoverride

Definition at line 118 of file GlobalFileMgr.h.

References logger::FATAL, and LOG.

118 { LOG(FATAL) << "Operation not supported"; }
#define LOG(tag)
Definition: Logger.h:285
size_t File_Namespace::GlobalFileMgr::getAllocated ( )
inlineoverride

Definition at line 125 of file GlobalFileMgr.h.

125 { return 0; }
std::string File_Namespace::GlobalFileMgr::getBasePath ( ) const
inline

Definition at line 170 of file GlobalFileMgr.h.

Referenced by anonymous_namespace{TableArchiver.cpp}::abs_path(), File_Namespace::FileMgr::coreInit(), and File_Namespace::FileMgr::FileMgr().

170 { return basePath_; }

+ Here is the caller graph for this function:

AbstractBuffer* File_Namespace::GlobalFileMgr::getBuffer ( const ChunkKey key,
const size_t  numBytes = 0 
)
inlineoverride

Returns the a pointer to the chunk with the specified key.

Definition at line 90 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint().

90  {
91  return getFileMgr(key)->getBuffer(key, numBytes);
92  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunkMetadataVec,
const ChunkKey keyPrefix 
)
inlineoverride

Definition at line 130 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix().

131  {
132  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
133  keyPrefix);
134  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the caller graph for this function:

bool File_Namespace::GlobalFileMgr::getDBConvert ( ) const
inline

Definition at line 177 of file GlobalFileMgr.h.

Referenced by File_Namespace::FileMgr::getDBConvert().

177 { return dbConvert_; }

+ Here is the caller graph for this function:

int32_t File_Namespace::GlobalFileMgr::getDBVersion ( ) const
inline

Definition at line 176 of file GlobalFileMgr.h.

Referenced by File_Namespace::FileMgr::getDBVersion().

176 { return omnisci_db_version_; }
int32_t omnisci_db_version_
used to set FileMgr metadta_page_size_

+ Here is the caller graph for this function:

AbstractBufferMgr * File_Namespace::GlobalFileMgr::getFileMgr ( const int32_t  db_id,
const int32_t  tb_id 
)

Definition at line 180 of file GlobalFileMgr.cpp.

References allFileMgrs_, CHECK, epoch_, fileMgrs_mutex_, findFileMgrUnlocked(), fsi_, lazy_initialized_stats_, max_rollback_epochs_per_table_, num_reader_threads_, and ownedFileMgrs_.

Referenced by checkpoint(), compactDataFiles(), deleteBuffersWithPrefix(), File_Namespace::FileMgr::init(), and resetTableEpochFloor().

180  {
181  { // check if FileMgr already exists for (db_id, tb_id)
183  AbstractBufferMgr* fm = findFileMgrUnlocked(db_id, tb_id);
184  if (fm) {
185  return fm;
186  }
187  }
188 
189  { // create new FileMgr for (db_id, tb_id)
191  AbstractBufferMgr* fm = findFileMgrUnlocked(db_id, tb_id);
192  if (fm) {
193  return fm; // mgr was added between the read lock and the write lock
194  }
195  const auto file_mgr_key = std::make_pair(db_id, tb_id);
196  const auto foreign_buffer_manager = fsi_->lookupBufferManager(db_id, tb_id);
197  if (foreign_buffer_manager) {
198  CHECK(allFileMgrs_.insert(std::make_pair(file_mgr_key, foreign_buffer_manager))
199  .second);
200  return foreign_buffer_manager;
201  } else {
202  int32_t max_rollback_epochs{-1};
203  if (max_rollback_epochs_per_table_.find(file_mgr_key) !=
205  max_rollback_epochs = max_rollback_epochs_per_table_[file_mgr_key];
206  }
207  auto s = std::make_shared<FileMgr>(
208  0, this, file_mgr_key, max_rollback_epochs, num_reader_threads_, epoch_);
209  CHECK(ownedFileMgrs_.insert(std::make_pair(file_mgr_key, s)).second);
210  CHECK(allFileMgrs_.insert(std::make_pair(file_mgr_key, s.get())).second);
211  lazy_initialized_stats_.erase(file_mgr_key);
212  return s.get();
213  }
214  }
215 }
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::shared_lock< T > shared_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
AbstractBufferMgr * findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id)
std::unique_lock< T > unique_lock
size_t num_reader_threads_
The OS file system path containing the files.
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
#define CHECK(condition)
Definition: Logger.h:291
std::map< TablePair, StorageStats > lazy_initialized_stats_
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBufferMgr* File_Namespace::GlobalFileMgr::getFileMgr ( const ChunkKey key)
inline

Definition at line 166 of file GlobalFileMgr.h.

166  {
167  return getFileMgr(key[0], key[1]);
168  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
size_t File_Namespace::GlobalFileMgr::getInUseSize ( )
inlineoverride

Definition at line 124 of file GlobalFileMgr.h.

124 { return 0; }
size_t File_Namespace::GlobalFileMgr::getMaxSize ( )
inlineoverride

Definition at line 123 of file GlobalFileMgr.h.

123 { return 0; }
size_t File_Namespace::GlobalFileMgr::getMetadataPageSize ( ) const
inline

Definition at line 172 of file GlobalFileMgr.h.

172 { return metadata_page_size_; }
const size_t metadata_page_size_
used to set FileMgr page_size_
MgrType File_Namespace::GlobalFileMgr::getMgrType ( )
inlineoverride

Definition at line 120 of file GlobalFileMgr.h.

120 { return GLOBAL_FILE_MGR; };
size_t File_Namespace::GlobalFileMgr::getNumChunks ( )
override

Definition at line 91 of file GlobalFileMgr.cpp.

References allFileMgrs_, and fileMgrs_mutex_.

91  {
93  size_t num_chunks = 0;
94  for (auto fileMgrsIt = allFileMgrs_.begin(); fileMgrsIt != allFileMgrs_.end();
95  ++fileMgrsIt) {
96  num_chunks += fileMgrsIt->second->getNumChunks();
97  }
98 
99  return num_chunks;
100 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::shared_lock< T > shared_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
heavyai::shared_mutex fileMgrs_mutex_
size_t File_Namespace::GlobalFileMgr::getNumReaderThreads ( )
inline

Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.

Definition at line 147 of file GlobalFileMgr.h.

147 { return num_reader_threads_; }
size_t num_reader_threads_
The OS file system path containing the files.
size_t File_Namespace::GlobalFileMgr::getPageSize ( ) const
inline

Definition at line 171 of file GlobalFileMgr.h.

171 { return page_size_; }
std::shared_ptr< FileMgr > File_Namespace::GlobalFileMgr::getSharedFileMgr ( const int  db_id,
const int  table_id 
)

Definition at line 218 of file GlobalFileMgr.cpp.

References ownedFileMgrs_.

219  {
220  const auto table_key = std::make_pair(db_id, table_id);
221  if (ownedFileMgrs_.find(table_key) == ownedFileMgrs_.end()) {
222  return nullptr;
223  }
224  return ownedFileMgrs_[table_key];
225 }
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
StorageStats File_Namespace::GlobalFileMgr::getStorageStats ( const int32_t  db_id,
const int32_t  tb_id 
)

Definition at line 317 of file GlobalFileMgr.cpp.

References fileMgrs_mutex_, findFileMgr(), and lazy_initialized_stats_.

317  {
319  AbstractBufferMgr* opened_fm = findFileMgr(db_id, tb_id);
320  if (opened_fm) {
321  return dynamic_cast<FileMgr*>(opened_fm)->getStorageStats();
322  }
323  TablePair file_mgr_key{db_id, tb_id};
324  auto it = lazy_initialized_stats_.find(file_mgr_key);
325  if (it != lazy_initialized_stats_.end()) {
326  return it->second;
327  } else {
328  // Do not do full init of table just to get storage stats, just check file instead
329  auto u = std::make_unique<FileMgr>(0, this, file_mgr_key, true);
330  lazy_initialized_stats_[file_mgr_key] = u->getStorageStats();
331  u.reset();
332  return lazy_initialized_stats_[file_mgr_key];
333  }
334 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::shared_lock< T > shared_lock
StorageStats getStorageStats(const int32_t db_id, const int32_t tb_id)
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
std::map< TablePair, StorageStats > lazy_initialized_stats_
std::pair< const int32_t, const int32_t > TablePair
Definition: FileMgr.h:91
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the call graph for this function:

std::string File_Namespace::GlobalFileMgr::getStringMgrType ( )
inlineoverride

Definition at line 121 of file GlobalFileMgr.h.

121 { return ToString(GLOBAL_FILE_MGR); }
size_t File_Namespace::GlobalFileMgr::getTableEpoch ( const int32_t  db_id,
const int32_t  tb_id 
)

Definition at line 295 of file GlobalFileMgr.cpp.

References fileMgrs_mutex_, and findFileMgr().

295  {
296  // UX change was made to this function Oct 2020 to return checkpointed epoch. In turn,
297  // setTableEpoch was changed to set the epoch at the user's input, instead of input - 1
299  AbstractBufferMgr* opened_fm = findFileMgr(db_id, tb_id);
300  if (opened_fm) {
301  return dynamic_cast<FileMgr*>(opened_fm)->lastCheckpointedEpoch();
302  }
303  // Do not do full init of table just to get table epoch, just check file instead
304  const auto file_mgr_key = std::make_pair(db_id, tb_id);
305  auto u = std::make_unique<FileMgr>(0, this, file_mgr_key, true);
306  const auto epoch = u->lastCheckpointedEpoch();
307  u.reset();
308  return epoch;
309 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::shared_lock< T > shared_lock
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::init ( )

Definition at line 62 of file GlobalFileMgr.cpp.

References basePath_, logger::FATAL, and LOG.

Referenced by GlobalFileMgr().

62  {
63  // check if basePath_ already exists, and if not create one
64  boost::filesystem::path path(basePath_);
65  if (basePath_.size() > 0 && basePath_[basePath_.size() - 1] != '/') {
66  basePath_.push_back('/');
67  }
68  if (boost::filesystem::exists(path)) {
69  if (!boost::filesystem::is_directory(path)) {
70  LOG(FATAL) << "Specified path is not a directory.";
71  }
72  } else { // data directory does not exist
73  if (!boost::filesystem::create_directory(path)) {
74  LOG(FATAL) << "Could not create data directory";
75  }
76  }
77 }
#define LOG(tag)
Definition: Logger.h:285

+ Here is the caller graph for this function:

bool File_Namespace::GlobalFileMgr::isAllocationCapped ( )
inlineoverride

Definition at line 126 of file GlobalFileMgr.h.

126 { return false; }
bool File_Namespace::GlobalFileMgr::isBufferOnDevice ( const ChunkKey key)
inlineoverride

Definition at line 74 of file GlobalFileMgr.h.

74  {
75  return getFileMgr(key)->isBufferOnDevice(key);
76  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
std::string File_Namespace::GlobalFileMgr::printSlabs ( )
inlineoverride

Definition at line 122 of file GlobalFileMgr.h.

122 { return "Not Implemented"; }
AbstractBuffer* File_Namespace::GlobalFileMgr::putBuffer ( const ChunkKey key,
AbstractBuffer d,
const size_t  numBytes = 0 
)
inlineoverride

Puts the contents of d into the Chunk with the given key.

Parameters
key- Unique identifier for a Chunk.
d- An object representing the source data for the Chunk.
Returns
AbstractBuffer*

Definition at line 106 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::putBuffer().

108  {
109  return getFileMgr(key)->putBuffer(key, d, numBytes);
110  }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::removeTableRelatedDS ( const int32_t  db_id,
const int32_t  tb_id 
)
override

Definition at line 254 of file GlobalFileMgr.cpp.

References deleteFileMgr(), fileMgrs_mutex_, findFileMgrUnlocked(), fsi_, and max_rollback_epochs_per_table_.

Referenced by File_Namespace::CachingGlobalFileMgr::removeTableRelatedDS().

254  {
256  auto abm = findFileMgrUnlocked(db_id, tb_id);
257  if (auto fm = dynamic_cast<File_Namespace::FileMgr*>(abm)) {
258  fm->closeRemovePhysical();
259  } else if (dynamic_cast<ForeignStorageBufferMgr*>(abm)) {
260  abm->removeTableRelatedDS(db_id, tb_id);
261  fsi_->dropBufferManager(db_id, tb_id);
262  } else {
263  // fileMgr has not been initialized so there is no need to
264  // spend the time initializing
265  // initialize just enough to have to rename
266  const auto file_mgr_key = std::make_pair(db_id, tb_id);
267  auto u = std::make_unique<FileMgr>(0, this, file_mgr_key, true);
268  u->closeRemovePhysical();
269  }
270  // remove table related in-memory DS only if directory was removed successfully
271 
272  deleteFileMgr(db_id, tb_id);
273  max_rollback_epochs_per_table_.erase({db_id, tb_id});
274 }
void deleteFileMgr(const int32_t db_id, const int32_t tb_id)
std::shared_ptr< ForeignStorageInterface > fsi_
heavyai::unique_lock< heavyai::shared_mutex > write_lock
AbstractBufferMgr * findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id)
std::unique_lock< T > unique_lock
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::resetTableEpochFloor ( const int32_t  db_id,
const int32_t  tb_id 
)

Definition at line 311 of file GlobalFileMgr.cpp.

References CHECK, and getFileMgr().

311  {
312  AbstractBufferMgr* fm = getFileMgr(db_id, tb_id);
313  CHECK(fm);
314  dynamic_cast<FileMgr*>(fm)->resetEpochFloor();
315 }
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::setDBConvert ( bool  val)
inline

Definition at line 178 of file GlobalFileMgr.h.

void File_Namespace::GlobalFileMgr::setFileMgr ( const int  db_id,
const int  table_id,
std::shared_ptr< FileMgr file_mgr 
)

Definition at line 228 of file GlobalFileMgr.cpp.

References allFileMgrs_, lazy_initialized_stats_, and ownedFileMgrs_.

230  {
231  TablePair file_mgr_key{db_id, table_id};
232  allFileMgrs_[file_mgr_key] = file_mgr.get();
233  ownedFileMgrs_[file_mgr_key] = file_mgr;
234  lazy_initialized_stats_.erase(file_mgr_key);
235 }
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
std::map< TablePair, StorageStats > lazy_initialized_stats_
std::pair< const int32_t, const int32_t > TablePair
Definition: FileMgr.h:91
void File_Namespace::GlobalFileMgr::setFileMgrParams ( const int32_t  db_id,
const int32_t  tb_id,
const FileMgrParams file_mgr_params 
)

Definition at line 154 of file GlobalFileMgr.cpp.

References allFileMgrs_, CHECK, deleteFileMgr(), File_Namespace::FileMgrParams::epoch, epoch_, fileMgrs_mutex_, findFileMgr(), lazy_initialized_stats_, File_Namespace::FileMgrParams::max_rollback_epochs, max_rollback_epochs_per_table_, num_reader_threads_, and ownedFileMgrs_.

156  {
157  auto fm = dynamic_cast<File_Namespace::FileMgr*>(findFileMgr(db_id, tb_id));
159  if (fm) {
160  deleteFileMgr(db_id, tb_id);
161  }
162  const auto file_mgr_key = std::make_pair(db_id, tb_id);
163  auto max_rollback_epochs =
164  (file_mgr_params.max_rollback_epochs >= 0 ? file_mgr_params.max_rollback_epochs
165  : -1);
166  auto s = std::make_shared<FileMgr>(
167  0,
168  this,
169  file_mgr_key,
170  max_rollback_epochs,
172  file_mgr_params.epoch != -1 ? file_mgr_params.epoch : epoch_);
173  CHECK(ownedFileMgrs_.insert(std::make_pair(file_mgr_key, s)).second);
174  CHECK(allFileMgrs_.insert(std::make_pair(file_mgr_key, s.get())).second);
175  max_rollback_epochs_per_table_[file_mgr_key] = max_rollback_epochs;
176  lazy_initialized_stats_.erase(file_mgr_key);
177  return;
178 }
void deleteFileMgr(const int32_t db_id, const int32_t tb_id)
int32_t epoch_
number of threads used when loading data
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
std::unique_lock< T > unique_lock
size_t num_reader_threads_
The OS file system path containing the files.
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
#define CHECK(condition)
Definition: Logger.h:291
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
std::map< TablePair, StorageStats > lazy_initialized_stats_
heavyai::shared_mutex fileMgrs_mutex_

+ Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::setTableEpoch ( const int32_t  db_id,
const int32_t  tb_id,
const int32_t  start_epoch 
)

Definition at line 276 of file GlobalFileMgr.cpp.

References deleteFileMgr(), findFileMgr(), and num_reader_threads_.

278  {
279  AbstractBufferMgr* opened_fm = findFileMgr(db_id, tb_id);
280  if (opened_fm) {
281  // Delete this FileMgr to ensure epoch change occurs in constructor with other
282  // reads/writes locked out
283  deleteFileMgr(db_id, tb_id);
284  }
285  const auto file_mgr_key = std::make_pair(db_id, tb_id);
286  // this is where the real rollback of any data ahead of the currently set epoch is
287  // performed
288  // Will call set_epoch with start_epoch internally
289  auto u = std::make_unique<FileMgr>(
290  0, this, file_mgr_key, -1, num_reader_threads_, start_epoch);
291  // remove the dummy one we built
292  u.reset();
293 }
void deleteFileMgr(const int32_t db_id, const int32_t tb_id)
size_t num_reader_threads_
The OS file system path containing the files.
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)

+ Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::writeFileMgrData ( FileMgr fileMgr = 0)

Definition at line 237 of file GlobalFileMgr.cpp.

References allFileMgrs_, CHECK, File_Namespace::FileMgr::chunkIndex_, and fileMgrs_mutex_.

238  { // this function is not used, keep it for now for future needs
240  for (auto fileMgrIt = allFileMgrs_.begin(); fileMgrIt != allFileMgrs_.end();
241  fileMgrIt++) {
242  FileMgr* fm = dynamic_cast<FileMgr*>(fileMgrIt->second);
243  CHECK(fm);
244  if ((fileMgr != 0) && (fileMgr != fm)) {
245  continue;
246  }
247  for (auto chunkIt = fm->chunkIndex_.begin(); chunkIt != fm->chunkIndex_.end();
248  chunkIt++) {
249  chunkIt->second->write((int8_t*)chunkIt->second, chunkIt->second->size(), 0);
250  }
251  }
252 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::shared_lock< T > shared_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
#define CHECK(condition)
Definition: Logger.h:291
heavyai::shared_mutex fileMgrs_mutex_

Member Data Documentation

std::map<TablePair, AbstractBufferMgr*> File_Namespace::GlobalFileMgr::allFileMgrs_
private
std::string File_Namespace::GlobalFileMgr::basePath_
private

Definition at line 201 of file GlobalFileMgr.h.

Referenced by init().

bool File_Namespace::GlobalFileMgr::dbConvert_
private

DB version for DataMgr DS and corresponding file buffer read/write code

Definition at line 218 of file GlobalFileMgr.h.

Referenced by GlobalFileMgr().

int32_t File_Namespace::GlobalFileMgr::epoch_
private

number of threads used when loading data

Definition at line 204 of file GlobalFileMgr.h.

Referenced by getFileMgr(), and setFileMgrParams().

std::shared_ptr<ForeignStorageInterface> File_Namespace::GlobalFileMgr::fsi_
protected
std::map<TablePair, StorageStats> File_Namespace::GlobalFileMgr::lazy_initialized_stats_
private

Definition at line 224 of file GlobalFileMgr.h.

Referenced by getFileMgr(), getStorageStats(), setFileMgr(), and setFileMgrParams().

std::map<TablePair, int32_t> File_Namespace::GlobalFileMgr::max_rollback_epochs_per_table_
private

Definition at line 223 of file GlobalFileMgr.h.

Referenced by getFileMgr(), removeTableRelatedDS(), and setFileMgrParams().

const size_t File_Namespace::GlobalFileMgr::metadata_page_size_
private

used to set FileMgr page_size_

Definition at line 209 of file GlobalFileMgr.h.

size_t File_Namespace::GlobalFileMgr::num_reader_threads_
private

The OS file system path containing the files.

Definition at line 202 of file GlobalFileMgr.h.

Referenced by getFileMgr(), setFileMgrParams(), and setTableEpoch().

int32_t File_Namespace::GlobalFileMgr::omnisci_db_version_
private

used to set FileMgr metadta_page_size_

Definition at line 212 of file GlobalFileMgr.h.

Referenced by GlobalFileMgr().

std::map<TablePair, std::shared_ptr<FileMgr> > File_Namespace::GlobalFileMgr::ownedFileMgrs_
private

true if conversion should be done between different "omnisci_db_version_"

Definition at line 221 of file GlobalFileMgr.h.

Referenced by deleteFileMgr(), getFileMgr(), getSharedFileMgr(), setFileMgr(), and setFileMgrParams().

const size_t File_Namespace::GlobalFileMgr::page_size_
private

Definition at line 208 of file GlobalFileMgr.h.


The documentation for this class was generated from the following files: