File_Namespace::GlobalFileMgr Class Reference

#include <GlobalFileMgr.h>

Inheritance diagram for File_Namespace::GlobalFileMgr:

Collaboration diagram for File_Namespace::GlobalFileMgr:

Public Member Functions
	GlobalFileMgr (const int32_t device_id, std::shared_ptr< ForeignStorageInterface > fsi, std::string base_path=".", const size_t num_reader_threads=0, const size_t page_size=DEFAULT_PAGE_SIZE, const size_t metadata_page_size=DEFAULT_METADATA_PAGE_SIZE)
	Constructor. More...
	~GlobalFileMgr () override
AbstractBuffer *	createBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
	Creates a chunk with the specified key and page size. More...
bool	isBufferOnDevice (const ChunkKey &key) override
void	deleteBuffer (const ChunkKey &key, const bool purge=true) override
	Deletes the chunk with the specified key. More...
void	deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
AbstractBuffer *	getBuffer (const ChunkKey &key, const size_t numBytes=0) override
	Returns the a pointer to the chunk with the specified key. More...
void	fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
AbstractBuffer *	putBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
	Puts the contents of d into the Chunk with the given key. More...
AbstractBuffer *	alloc (const size_t numBytes) override
void	free (AbstractBuffer *buffer) override
MgrType	getMgrType () override
std::string	getStringMgrType () override
std::string	printSlabs () override
size_t	getMaxSize () override
size_t	getInUseSize () override
size_t	getAllocated () override
bool	isAllocationCapped () override
void	init ()
void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
void	checkpoint () override
	Fsyncs data files, writes out epoch and fsyncs that. More...
void	checkpoint (const int32_t db_id, const int32_t tb_id) override
size_t	getNumReaderThreads ()
	Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...
size_t	getNumChunks () override
void	compactDataFiles (const int32_t db_id, const int32_t tb_id)
AbstractBufferMgr *	findFileMgr (const int32_t db_id, const int32_t tb_id)
void	setFileMgrParams (const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params)
AbstractBufferMgr *	getFileMgr (const int32_t db_id, const int32_t tb_id)
AbstractBufferMgr *	getFileMgr (const ChunkKey &key)
std::string	getBasePath () const
size_t	getPageSize () const
size_t	getMetadataPageSize () const
void	writeFileMgrData (FileMgr *fileMgr=0)
bool	getDBConvert () const
void	setDBConvert (bool val)
void	removeTableRelatedDS (const int32_t db_id, const int32_t tb_id) override
void	setTableEpoch (const int32_t db_id, const int32_t tb_id, const int32_t start_epoch)
size_t	getTableEpoch (const int32_t db_id, const int32_t tb_id)
void	resetTableEpochFloor (const int32_t db_id, const int32_t tb_id)
StorageStats	getStorageStats (const int32_t db_id, const int32_t tb_id)
std::shared_ptr< FileMgr >	getSharedFileMgr (const int db_id, const int table_id)
void	setFileMgr (const int db_id, const int table_id, std::shared_ptr< FileMgr > file_mgr)
void	closeFileMgr (const int32_t db_id, const int32_t tb_id)

Static Public Attributes
static constexpr int32_t	db_version_ {2}

Protected Attributes
std::shared_ptr < ForeignStorageInterface >	fsi_

Private Member Functions
AbstractBufferMgr *	findFileMgrUnlocked (const int32_t db_id, const int32_t tb_id)
void	deleteFileMgr (const int32_t db_id, const int32_t tb_id)
bool	existsDiffBetweenFileMgrParamsAndFileMgr (FileMgr *file_mgr, const FileMgrParams &file_mgr_params) const

Private Attributes
std::string	basePath_
size_t	num_reader_threads_
	The OS file system path containing the files. More...
int32_t	epoch_
	number of threads used when loading data More...
const size_t	page_size_
const size_t	metadata_page_size_
	used to set FileMgr page_size_ More...
bool	dbConvert_
	used to set FileMgr metadta_page_size_ More...
std::map< TablePair, std::shared_ptr< FileMgr > >	ownedFileMgrs_
std::map< TablePair, AbstractBufferMgr * >	allFileMgrs_
std::map< TablePair, int32_t >	max_rollback_epochs_per_table_
std::map< TablePair, StorageStats >	lazy_initialized_stats_
heavyai::shared_mutex	fileMgrs_mutex_

Detailed Description

Definition at line 52 of file GlobalFileMgr.h.

Constructor & Destructor Documentation

File_Namespace::GlobalFileMgr::GlobalFileMgr	(	const int32_t	device_id,
		std::shared_ptr< ForeignStorageInterface >	fsi,
		std::string	base_path = ".",
		const size_t	num_reader_threads = 0,
		const size_t	page_size = DEFAULT_PAGE_SIZE,
		const size_t	metadata_page_size = DEFAULT_METADATA_PAGE_SIZE
	)

Constructor.

Definition at line 42 of file GlobalFileMgr.cpp.

References dbConvert_, and init().

    : AbstractBufferMgr(device_id)
    , fsi_(fsi)
    , basePath_(base_path)
    , num_reader_threads_(num_reader_threads)
    , epoch_(-1)  // set the default epoch for all tables corresponding to the time of
                  // last checkpoint
    , page_size_(page_size)
    , metadata_page_size_(metadata_page_size) {
  // DS changes also triggered by individual FileMgr per table project (release 2.1.0)
  dbConvert_ = false;
  init();
}

Here is the call graph for this function:

File_Namespace::GlobalFileMgr::~GlobalFileMgr ( )

inlineoverride

Definition at line 63 of file GlobalFileMgr.h.

{}

Member Function Documentation

AbstractBuffer* File_Namespace::GlobalFileMgr::alloc ( const size_t  numBytes )

inlineoverride

Definition at line 111 of file GlobalFileMgr.h.

References logger::FATAL, and LOG.

                                                        {
    LOG(FATAL) << "Operation not supported";
    return nullptr;  // satisfy return-type warning
  }

void File_Namespace::GlobalFileMgr::checkpoint ( )

override

Fsyncs data files, writes out epoch and fsyncs that.

Definition at line 78 of file GlobalFileMgr.cpp.

References allFileMgrs_, and fileMgrs_mutex_.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint().

                               {
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(fileMgrs_mutex_);
  for (auto fileMgrsIt = allFileMgrs_.begin(); fileMgrsIt != allFileMgrs_.end();
       ++fileMgrsIt) {
    fileMgrsIt->second->checkpoint();
  }
}

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::checkpoint ( const int32_t  db_id, const int32_t  tb_id  )

override

Definition at line 86 of file GlobalFileMgr.cpp.

References getFileMgr().

                                                                       {
  getFileMgr(db_id, tb_id)->checkpoint();
}

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::closeFileMgr	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 134 of file GlobalFileMgr.cpp.

References deleteFileMgr(), and fileMgrs_mutex_.

                                                                         {
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(fileMgrs_mutex_);
  deleteFileMgr(db_id, tb_id);
}

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::compactDataFiles	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 335 of file GlobalFileMgr.cpp.

References deleteFileMgr(), fileMgrs_mutex_, findFileMgr(), and getFileMgr().

Referenced by TableOptimizer::vacuumDeletedRows().

                                                                             {
  auto file_mgr = dynamic_cast<File_Namespace::FileMgr*>(findFileMgr(db_id, tb_id));
  {
    heavyai::unique_lock<heavyai::shared_mutex> write_lock(fileMgrs_mutex_);
    if (file_mgr) {
      file_mgr->compactFiles();
      deleteFileMgr(db_id, tb_id);
    }
  }

  // Re-initialize file manager
  getFileMgr(db_id, tb_id);
}

Here is the call graph for this function:

Here is the caller graph for this function:

AbstractBuffer* File_Namespace::GlobalFileMgr::createBuffer ( const ChunkKey &  key, size_t  pageSize = 0, const size_t  numBytes = 0  )

inlineoverride

Creates a chunk with the specified key and page size.

Definition at line 66 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::createBuffer().

                                                                   {
    return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
  }

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::deleteBuffer ( const ChunkKey &  key, const bool  purge = true  )

inlineoverride

Deletes the chunk with the specified key.

Definition at line 80 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffer().

                                                                           {
    return getFileMgr(key)->deleteBuffer(key, purge);
  }

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::deleteBuffersWithPrefix ( const ChunkKey &  keyPrefix, const bool  purge = true  )

override

Definition at line 101 of file GlobalFileMgr.cpp.

References getFileMgr().

Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffersWithPrefix().

                                                                                       {
  /* keyPrefix[0] can be -1 only for gpu or cpu buffers but not for FileMgr.
   * There is no assert here, as GlobalFileMgr is being called with -1 value as well in
   * the same loop with other buffers. So the case of -1 will just be ignored, as nothing
   * needs to be done.
   */
  if (keyPrefix[0] != -1) {
    return getFileMgr(keyPrefix)->deleteBuffersWithPrefix(keyPrefix, purge);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::deleteFileMgr ( const int32_t  db_id, const int32_t  tb_id  )

private

Definition at line 123 of file GlobalFileMgr.cpp.

References allFileMgrs_, and ownedFileMgrs_.

Referenced by closeFileMgr(), compactDataFiles(), removeTableRelatedDS(), setFileMgrParams(), and setTableEpoch().

                                                                          {
  // NOTE: only call this private function after locking is already in place
  const auto file_mgr_key = std::make_pair(db_id, tb_id);
  if (auto it = ownedFileMgrs_.find(file_mgr_key); it != ownedFileMgrs_.end()) {
    ownedFileMgrs_.erase(it);
  }
  if (auto it = allFileMgrs_.find(file_mgr_key); it != allFileMgrs_.end()) {
    allFileMgrs_.erase(it);
  }
}

Here is the caller graph for this function:

bool File_Namespace::GlobalFileMgr::existsDiffBetweenFileMgrParamsAndFileMgr ( FileMgr *  file_mgr, const FileMgrParams &  file_mgr_params  ) const

private

Definition at line 139 of file GlobalFileMgr.cpp.

References File_Namespace::FileMgrParams::epoch, File_Namespace::FileMgr::lastCheckpointedEpoch(), File_Namespace::FileMgrParams::max_rollback_epochs, and File_Namespace::FileMgr::maxRollbackEpochs().

                                                {
  if (file_mgr_params.epoch != -1 &&
      file_mgr_params.epoch != file_mgr->lastCheckpointedEpoch()) {
    return true;
  }
  if (file_mgr_params.max_rollback_epochs != -1 &&
      file_mgr_params.max_rollback_epochs != file_mgr->maxRollbackEpochs()) {
    return true;
  }
  return false;
}

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::fetchBuffer ( const ChunkKey &  key, AbstractBuffer *  destBuffer, const size_t  numBytes  )

inlineoverride

Definition at line 92 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint(), and File_Namespace::CachingGlobalFileMgr::fetchBuffer().

                                                   {
    return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
  }

Here is the caller graph for this function:

AbstractBufferMgr* File_Namespace::GlobalFileMgr::findFileMgr ( const int32_t  db_id, const int32_t  tb_id  )

inline

Definition at line 158 of file GlobalFileMgr.h.

Referenced by compactDataFiles(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), getStorageStats(), getTableEpoch(), setFileMgrParams(), and setTableEpoch().

                                                                           {
    heavyai::shared_lock<heavyai::shared_mutex> read_lock(fileMgrs_mutex_);
    return findFileMgrUnlocked(db_id, tb_id);
  }

Here is the caller graph for this function:

AbstractBufferMgr * File_Namespace::GlobalFileMgr::findFileMgrUnlocked ( const int32_t  db_id, const int32_t  tb_id  )

private

Definition at line 112 of file GlobalFileMgr.cpp.

References allFileMgrs_.

Referenced by getFileMgr(), and removeTableRelatedDS().

                                                                           {
  // NOTE: only call this private function after locking is already in place
  AbstractBufferMgr* fm = nullptr;
  const auto file_mgr_key = std::make_pair(db_id, tb_id);
  if (auto it = allFileMgrs_.find(file_mgr_key); it != allFileMgrs_.end()) {
    fm = it->second;
  }
  return fm;
}

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::free ( AbstractBuffer *  buffer )

inlineoverride

Definition at line 116 of file GlobalFileMgr.h.

References logger::FATAL, and LOG.

{ LOG(FATAL) << "Operation not supported"; }

size_t File_Namespace::GlobalFileMgr::getAllocated ( )

inlineoverride

Definition at line 123 of file GlobalFileMgr.h.

{ return 0; }

std::string File_Namespace::GlobalFileMgr::getBasePath ( ) const

inline

Definition at line 170 of file GlobalFileMgr.h.

Referenced by anonymous_namespace{TableArchiver.cpp}::abs_path(), File_Namespace::FileMgr::coreInit(), and File_Namespace::FileMgr::FileMgr().

{ return basePath_; }

Here is the caller graph for this function:

AbstractBuffer* File_Namespace::GlobalFileMgr::getBuffer ( const ChunkKey &  key, const size_t  numBytes = 0  )

inlineoverride

Returns the a pointer to the chunk with the specified key.

Definition at line 88 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint().

                                                                                     {
    return getFileMgr(key)->getBuffer(key, numBytes);
  }

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector &  chunkMetadataVec, const ChunkKey &  keyPrefix  )

inlineoverride

Definition at line 128 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix().

                                                                           {
    return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
                                                                  keyPrefix);
  }

Here is the caller graph for this function:

bool File_Namespace::GlobalFileMgr::getDBConvert ( ) const

inline

Definition at line 176 of file GlobalFileMgr.h.

Referenced by File_Namespace::FileMgr::getDBConvert().

{ return dbConvert_; }

Here is the caller graph for this function:

AbstractBufferMgr * File_Namespace::GlobalFileMgr::getFileMgr	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 179 of file GlobalFileMgr.cpp.

References allFileMgrs_, CHECK, epoch_, fileMgrs_mutex_, findFileMgrUnlocked(), fsi_, lazy_initialized_stats_, max_rollback_epochs_per_table_, num_reader_threads_, and ownedFileMgrs_.

Referenced by checkpoint(), compactDataFiles(), deleteBuffersWithPrefix(), File_Namespace::FileMgr::init(), and resetTableEpochFloor().

                                                                                     {
  {  // check if FileMgr already exists for (db_id, tb_id)
    heavyai::shared_lock<heavyai::shared_mutex> read_lock(fileMgrs_mutex_);
    AbstractBufferMgr* fm = findFileMgrUnlocked(db_id, tb_id);
    if (fm) {
      return fm;
    }
  }

  {  // create new FileMgr for (db_id, tb_id)
    heavyai::unique_lock<heavyai::shared_mutex> write_lock(fileMgrs_mutex_);
    AbstractBufferMgr* fm = findFileMgrUnlocked(db_id, tb_id);
    if (fm) {
      return fm;  // mgr was added between the read lock and the write lock
    }
    const auto file_mgr_key = std::make_pair(db_id, tb_id);
    const auto foreign_buffer_manager = fsi_->lookupBufferManager(db_id, tb_id);
    if (foreign_buffer_manager) {
      CHECK(allFileMgrs_.insert(std::make_pair(file_mgr_key, foreign_buffer_manager))
                .second);
      return foreign_buffer_manager;
    } else {
      int32_t max_rollback_epochs{-1};
      if (max_rollback_epochs_per_table_.find(file_mgr_key) !=
          max_rollback_epochs_per_table_.end()) {
        max_rollback_epochs = max_rollback_epochs_per_table_[file_mgr_key];
      }
      auto s = std::make_shared<FileMgr>(
          0, this, file_mgr_key, max_rollback_epochs, num_reader_threads_, epoch_);
      CHECK(ownedFileMgrs_.insert(std::make_pair(file_mgr_key, s)).second);
      CHECK(allFileMgrs_.insert(std::make_pair(file_mgr_key, s.get())).second);
      lazy_initialized_stats_.erase(file_mgr_key);
      return s.get();
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

AbstractBufferMgr* File_Namespace::GlobalFileMgr::getFileMgr ( const ChunkKey &  key )

inline

Definition at line 166 of file GlobalFileMgr.h.

                                                     {
    return getFileMgr(key[0], key[1]);
  }

size_t File_Namespace::GlobalFileMgr::getInUseSize ( )

inlineoverride

Definition at line 122 of file GlobalFileMgr.h.

{ return 0; }

size_t File_Namespace::GlobalFileMgr::getMaxSize ( )

inlineoverride

Definition at line 121 of file GlobalFileMgr.h.

{ return 0; }

size_t File_Namespace::GlobalFileMgr::getMetadataPageSize ( ) const

inline

Definition at line 172 of file GlobalFileMgr.h.

{ return metadata_page_size_; }

MgrType File_Namespace::GlobalFileMgr::getMgrType ( )

inlineoverride

Definition at line 118 of file GlobalFileMgr.h.

{ return GLOBAL_FILE_MGR; };

size_t File_Namespace::GlobalFileMgr::getNumChunks ( )

override

Definition at line 90 of file GlobalFileMgr.cpp.

References allFileMgrs_, and fileMgrs_mutex_.

                                   {
  heavyai::shared_lock<heavyai::shared_mutex> read_lock(fileMgrs_mutex_);
  size_t num_chunks = 0;
  for (auto fileMgrsIt = allFileMgrs_.begin(); fileMgrsIt != allFileMgrs_.end();
       ++fileMgrsIt) {
    num_chunks += fileMgrsIt->second->getNumChunks();
  }

  return num_chunks;
}

size_t File_Namespace::GlobalFileMgr::getNumReaderThreads ( )

inline

Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data.

Definition at line 145 of file GlobalFileMgr.h.

{ return num_reader_threads_; }

size_t File_Namespace::GlobalFileMgr::getPageSize ( ) const

inline

Definition at line 171 of file GlobalFileMgr.h.

{ return page_size_; }

std::shared_ptr< FileMgr > File_Namespace::GlobalFileMgr::getSharedFileMgr	(	const int	db_id,
		const int	table_id
	)

Definition at line 217 of file GlobalFileMgr.cpp.

References ownedFileMgrs_.

                                                                             {
  const auto table_key = std::make_pair(db_id, table_id);
  if (ownedFileMgrs_.find(table_key) == ownedFileMgrs_.end()) {
    return nullptr;
  }
  return ownedFileMgrs_[table_key];
}

StorageStats File_Namespace::GlobalFileMgr::getStorageStats	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 316 of file GlobalFileMgr.cpp.

References fileMgrs_mutex_, findFileMgr(), and lazy_initialized_stats_.

                                                                                    {
  heavyai::shared_lock<heavyai::shared_mutex> read_lock(fileMgrs_mutex_);
  AbstractBufferMgr* opened_fm = findFileMgr(db_id, tb_id);
  if (opened_fm) {
    return dynamic_cast<FileMgr*>(opened_fm)->getStorageStats();
  }
  TablePair file_mgr_key{db_id, tb_id};
  auto it = lazy_initialized_stats_.find(file_mgr_key);
  if (it != lazy_initialized_stats_.end()) {
    return it->second;
  } else {
    // Do not do full init of table just to get storage stats, just check file instead
    auto u = std::make_unique<FileMgr>(0, this, file_mgr_key, true);
    lazy_initialized_stats_[file_mgr_key] = u->getStorageStats();
    u.reset();
    return lazy_initialized_stats_[file_mgr_key];
  }
}

Here is the call graph for this function:

std::string File_Namespace::GlobalFileMgr::getStringMgrType ( )

inlineoverride

Definition at line 119 of file GlobalFileMgr.h.

{ return ToString(GLOBAL_FILE_MGR); }

size_t File_Namespace::GlobalFileMgr::getTableEpoch	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 294 of file GlobalFileMgr.cpp.

References fileMgrs_mutex_, and findFileMgr().

                                                                            {
  // UX change was made to this function Oct 2020 to return checkpointed epoch. In turn,
  // setTableEpoch was changed to set the epoch at the user's input, instead of input - 1
  heavyai::shared_lock<heavyai::shared_mutex> read_lock(fileMgrs_mutex_);
  AbstractBufferMgr* opened_fm = findFileMgr(db_id, tb_id);
  if (opened_fm) {
    return dynamic_cast<FileMgr*>(opened_fm)->lastCheckpointedEpoch();
  }
  // Do not do full init of table just to get table epoch, just check file instead
  const auto file_mgr_key = std::make_pair(db_id, tb_id);
  auto u = std::make_unique<FileMgr>(0, this, file_mgr_key, true);
  const auto epoch = u->lastCheckpointedEpoch();
  u.reset();
  return epoch;
}

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::init

(

)

Definition at line 61 of file GlobalFileMgr.cpp.

References basePath_, logger::FATAL, and LOG.

Referenced by GlobalFileMgr().

                         {
  // check if basePath_ already exists, and if not create one
  boost::filesystem::path path(basePath_);
  if (basePath_.size() > 0 && basePath_[basePath_.size() - 1] != '/') {
    basePath_.push_back('/');
  }
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      LOG(FATAL) << "Specified path is not a directory.";
    }
  } else {  // data directory does not exist
    if (!boost::filesystem::create_directory(path)) {
      LOG(FATAL) << "Could not create data directory";
    }
  }
}

Here is the caller graph for this function:

bool File_Namespace::GlobalFileMgr::isAllocationCapped ( )

inlineoverride

Definition at line 124 of file GlobalFileMgr.h.

{ return false; }

bool File_Namespace::GlobalFileMgr::isBufferOnDevice ( const ChunkKey &  key )

inlineoverride

Definition at line 72 of file GlobalFileMgr.h.

                                                      {
    return getFileMgr(key)->isBufferOnDevice(key);
  }

std::string File_Namespace::GlobalFileMgr::printSlabs ( )

inlineoverride

Definition at line 120 of file GlobalFileMgr.h.

{ return "Not Implemented"; }

AbstractBuffer* File_Namespace::GlobalFileMgr::putBuffer ( const ChunkKey &  key, AbstractBuffer *  d, const size_t  numBytes = 0  )

inlineoverride

Puts the contents of d into the Chunk with the given key.

Parameters

key	- Unique identifier for a Chunk.
d	- An object representing the source data for the Chunk.

Returns

AbstractBuffer*

Definition at line 104 of file GlobalFileMgr.h.

Referenced by File_Namespace::CachingGlobalFileMgr::putBuffer().

                                                                {
    return getFileMgr(key)->putBuffer(key, d, numBytes);
  }

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::removeTableRelatedDS ( const int32_t  db_id, const int32_t  tb_id  )

override

Definition at line 253 of file GlobalFileMgr.cpp.

References deleteFileMgr(), fileMgrs_mutex_, findFileMgrUnlocked(), fsi_, and max_rollback_epochs_per_table_.

Referenced by File_Namespace::CachingGlobalFileMgr::removeTableRelatedDS().

                                                                                 {
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(fileMgrs_mutex_);
  auto abm = findFileMgrUnlocked(db_id, tb_id);
  if (auto fm = dynamic_cast<File_Namespace::FileMgr*>(abm)) {
    fm->closeRemovePhysical();
  } else if (dynamic_cast<ForeignStorageBufferMgr*>(abm)) {
    abm->removeTableRelatedDS(db_id, tb_id);
    fsi_->dropBufferManager(db_id, tb_id);
  } else {
    // fileMgr has not been initialized so there is no need to
    // spend the time initializing
    // initialize just enough to have to rename
    const auto file_mgr_key = std::make_pair(db_id, tb_id);
    auto u = std::make_unique<FileMgr>(0, this, file_mgr_key, true);
    u->closeRemovePhysical();
  }
  // remove table related in-memory DS only if directory was removed successfully

  deleteFileMgr(db_id, tb_id);
  max_rollback_epochs_per_table_.erase({db_id, tb_id});
}

Here is the call graph for this function:

Here is the caller graph for this function:

void File_Namespace::GlobalFileMgr::resetTableEpochFloor	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 310 of file GlobalFileMgr.cpp.

References CHECK, and getFileMgr().

                                                                                 {
  AbstractBufferMgr* fm = getFileMgr(db_id, tb_id);
  CHECK(fm);
  dynamic_cast<FileMgr*>(fm)->resetEpochFloor();
}

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::setDBConvert ( bool  val )

inline

Definition at line 177 of file GlobalFileMgr.h.

{ dbConvert_ = val; }

void File_Namespace::GlobalFileMgr::setFileMgr	(	const int	db_id,
		const int	table_id,
		std::shared_ptr< FileMgr >	file_mgr
	)

Definition at line 227 of file GlobalFileMgr.cpp.

References allFileMgrs_, lazy_initialized_stats_, and ownedFileMgrs_.

                                                                {
  TablePair file_mgr_key{db_id, table_id};
  allFileMgrs_[file_mgr_key] = file_mgr.get();
  ownedFileMgrs_[file_mgr_key] = file_mgr;
  lazy_initialized_stats_.erase(file_mgr_key);
}

void File_Namespace::GlobalFileMgr::setFileMgrParams	(	const int32_t	db_id,
		const int32_t	tb_id,
		const FileMgrParams &	file_mgr_params
	)

Definition at line 153 of file GlobalFileMgr.cpp.

References allFileMgrs_, CHECK, deleteFileMgr(), File_Namespace::FileMgrParams::epoch, epoch_, fileMgrs_mutex_, findFileMgr(), lazy_initialized_stats_, File_Namespace::FileMgrParams::max_rollback_epochs, max_rollback_epochs_per_table_, num_reader_threads_, and ownedFileMgrs_.

                                                                           {
  auto fm = dynamic_cast<File_Namespace::FileMgr*>(findFileMgr(db_id, tb_id));
  heavyai::unique_lock<heavyai::shared_mutex> write_lock(fileMgrs_mutex_);
  if (fm) {
    deleteFileMgr(db_id, tb_id);
  }
  const auto file_mgr_key = std::make_pair(db_id, tb_id);
  auto max_rollback_epochs =
      (file_mgr_params.max_rollback_epochs >= 0 ? file_mgr_params.max_rollback_epochs
                                                : -1);
  auto s = std::make_shared<FileMgr>(
      0,
      this,
      file_mgr_key,
      max_rollback_epochs,
      num_reader_threads_,
      file_mgr_params.epoch != -1 ? file_mgr_params.epoch : epoch_);
  CHECK(ownedFileMgrs_.insert(std::make_pair(file_mgr_key, s)).second);
  CHECK(allFileMgrs_.insert(std::make_pair(file_mgr_key, s.get())).second);
  max_rollback_epochs_per_table_[file_mgr_key] = max_rollback_epochs;
  lazy_initialized_stats_.erase(file_mgr_key);
  return;
}

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::setTableEpoch	(	const int32_t	db_id,
		const int32_t	tb_id,
		const int32_t	start_epoch
	)

Definition at line 275 of file GlobalFileMgr.cpp.

References deleteFileMgr(), findFileMgr(), and num_reader_threads_.

                                                             {
  AbstractBufferMgr* opened_fm = findFileMgr(db_id, tb_id);
  if (opened_fm) {
    // Delete this FileMgr to ensure epoch change occurs in constructor with other
    // reads/writes locked out
    deleteFileMgr(db_id, tb_id);
  }
  const auto file_mgr_key = std::make_pair(db_id, tb_id);
  // this is where the real rollback of any data ahead of the currently set epoch is
  // performed
  // Will call set_epoch with start_epoch internally
  auto u = std::make_unique<FileMgr>(
      0, this, file_mgr_key, -1, num_reader_threads_, start_epoch);
  // remove the dummy one we built
  u.reset();
}

Here is the call graph for this function:

void File_Namespace::GlobalFileMgr::writeFileMgrData

(

FileMgr *

fileMgr = 0

)

Definition at line 236 of file GlobalFileMgr.cpp.

References allFileMgrs_, CHECK, File_Namespace::FileMgr::chunkIndex_, and fileMgrs_mutex_.

                      {  // this function is not used, keep it for now for future needs
  heavyai::shared_lock<heavyai::shared_mutex> read_lock(fileMgrs_mutex_);
  for (auto fileMgrIt = allFileMgrs_.begin(); fileMgrIt != allFileMgrs_.end();
       fileMgrIt++) {
    FileMgr* fm = dynamic_cast<FileMgr*>(fileMgrIt->second);
    CHECK(fm);
    if ((fileMgr != 0) && (fileMgr != fm)) {
      continue;
    }
    for (auto chunkIt = fm->chunkIndex_.begin(); chunkIt != fm->chunkIndex_.end();
         chunkIt++) {
      chunkIt->second->write((int8_t*)chunkIt->second, chunkIt->second->size(), 0);
    }
  }
}

Member Data Documentation

std::map<TablePair, AbstractBufferMgr*> File_Namespace::GlobalFileMgr::allFileMgrs_

private

Definition at line 213 of file GlobalFileMgr.h.

Referenced by checkpoint(), deleteFileMgr(), findFileMgrUnlocked(), getFileMgr(), getNumChunks(), setFileMgr(), setFileMgrParams(), and writeFileMgrData().

std::string File_Namespace::GlobalFileMgr::basePath_

private

Definition at line 200 of file GlobalFileMgr.h.

Referenced by init().

constexpr int32_t File_Namespace::GlobalFileMgr::db_version_ {2}

static

Definition at line 151 of file GlobalFileMgr.h.

Referenced by File_Namespace::FileMgr::createOrMigrateTopLevelMetadata().

bool File_Namespace::GlobalFileMgr::dbConvert_

private

used to set FileMgr metadta_page_size_

Definition at line 209 of file GlobalFileMgr.h.

Referenced by GlobalFileMgr().

int32_t File_Namespace::GlobalFileMgr::epoch_

private

number of threads used when loading data

Definition at line 203 of file GlobalFileMgr.h.

Referenced by getFileMgr(), and setFileMgrParams().

heavyai::shared_mutex File_Namespace::GlobalFileMgr::fileMgrs_mutex_

private

Definition at line 217 of file GlobalFileMgr.h.

Referenced by checkpoint(), closeFileMgr(), compactDataFiles(), getFileMgr(), getNumChunks(), getStorageStats(), getTableEpoch(), removeTableRelatedDS(), setFileMgrParams(), and writeFileMgrData().

std::shared_ptr<ForeignStorageInterface> File_Namespace::GlobalFileMgr::fsi_

protected

Definition at line 194 of file GlobalFileMgr.h.

Referenced by getFileMgr(), File_Namespace::CachingGlobalFileMgr::isChunkPrefixCacheable(), and removeTableRelatedDS().

std::map<TablePair, StorageStats> File_Namespace::GlobalFileMgr::lazy_initialized_stats_

private

Definition at line 215 of file GlobalFileMgr.h.

Referenced by getFileMgr(), getStorageStats(), setFileMgr(), and setFileMgrParams().

std::map<TablePair, int32_t> File_Namespace::GlobalFileMgr::max_rollback_epochs_per_table_

private

Definition at line 214 of file GlobalFileMgr.h.

Referenced by getFileMgr(), removeTableRelatedDS(), and setFileMgrParams().

const size_t File_Namespace::GlobalFileMgr::metadata_page_size_

private

used to set FileMgr page_size_

Definition at line 208 of file GlobalFileMgr.h.

size_t File_Namespace::GlobalFileMgr::num_reader_threads_

private

The OS file system path containing the files.

Definition at line 201 of file GlobalFileMgr.h.

Referenced by getFileMgr(), setFileMgrParams(), and setTableEpoch().

std::map<TablePair, std::shared_ptr<FileMgr> > File_Namespace::GlobalFileMgr::ownedFileMgrs_

private

true if conversion should be done between different db_version

Definition at line 212 of file GlobalFileMgr.h.

Referenced by deleteFileMgr(), getFileMgr(), getSharedFileMgr(), setFileMgr(), and setFileMgrParams().

const size_t File_Namespace::GlobalFileMgr::page_size_

private

Definition at line 207 of file GlobalFileMgr.h.

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/GlobalFileMgr.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/FileMgr/GlobalFileMgr.cpp