#include <PersistentStorageMgr.h>

Inheritance diagram for PersistentStorageMgr:

Collaboration diagram for PersistentStorageMgr:

Public Member Functions
	PersistentStorageMgr (const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)

AbstractBuffer *	createBuffer (const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override

void	deleteBuffer (const ChunkKey &chunk_key, const bool purge) override

void	deleteBuffersWithPrefix (const ChunkKey &chunk_key_prefix, const bool purge) override

AbstractBuffer *	getBuffer (const ChunkKey &chunk_key, const size_t num_bytes) override

void	fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override

AbstractBuffer *	putBuffer (const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override

bool	isBufferOnDevice (const ChunkKey &chunk_key) override

std::string	printSlabs () override

void	clearSlabs () override

size_t	getMaxSize () override

size_t	getInUseSize () override

size_t	getAllocated () override

bool	isAllocationCapped () override

void	checkpoint () override

void	checkpoint (const int db_id, const int tb_id) override

AbstractBuffer *	alloc (const size_t num_bytes) override

void	free (AbstractBuffer *buffer) override

MgrType	getMgrType () override

std::string	getStringMgrType () override

size_t	getNumChunks () override

void	removeTableRelatedDS (const int db_id, const int table_id) override

File_Namespace::GlobalFileMgr *	getGlobalFileMgr () const

foreign_storage::ForeignStorageMgr *	getForeignStorageMgr () const

foreign_storage::ForeignStorageCache *	getDiskCache () const

const DiskCacheConfig	getDiskCacheConfig () const

Static Public Member Functions
static PersistentStorageMgr *	createPersistentStorageMgr (const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)

Protected Member Functions
bool	isForeignStorage (const ChunkKey &chunk_key) const

AbstractBufferMgr *	getStorageMgrForTableKey (const ChunkKey &table_key) const

bool	isChunkPrefixCacheable (const ChunkKey &chunk_prefix) const

int	recoverDataWrapperIfCachedAndGetHighestFragId (const ChunkKey &table_key)

Protected Attributes
std::unique_ptr < File_Namespace::GlobalFileMgr >	global_file_mgr_

std::unique_ptr < foreign_storage::ForeignStorageMgr >	foreign_storage_mgr_

std::unique_ptr < foreign_storage::ForeignStorageCache >	disk_cache_

DiskCacheConfig	disk_cache_config_

Detailed Description

Definition at line 26 of file PersistentStorageMgr.h.

Constructor & Destructor Documentation

PersistentStorageMgr::PersistentStorageMgr	(	const std::string &	data_dir,
		const size_t	num_reader_threads,
		const DiskCacheConfig &	disk_cache_config
	)

Definition at line 34 of file PersistentStorageMgr.cpp.

References disk_cache_, disk_cache_config_, foreign_storage_mgr_, DiskCacheConfig::isEnabled(), and DiskCacheConfig::isEnabledForFSI().

Referenced by createPersistentStorageMgr().

     : AbstractBufferMgr(0)
     , global_file_mgr_(
           std::make_unique<File_Namespace::GlobalFileMgr>(0,
                                                           data_dir,
                                                           num_reader_threads))
     , disk_cache_config_(disk_cache_config) {
   disk_cache_ =
       disk_cache_config_.isEnabled()
           ? std::make_unique<foreign_storage::ForeignStorageCache>(disk_cache_config)
           : nullptr;
   foreign_storage_mgr_ =
       disk_cache_config_.isEnabledForFSI()
           ? std::make_unique<foreign_storage::CachingForeignStorageMgr>(disk_cache_.get())
           : std::make_unique<foreign_storage::ForeignStorageMgr>();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Member Function Documentation

AbstractBuffer * PersistentStorageMgr::alloc ( const size_t num_bytes )

override

Definition at line 166 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                                                   {
   return global_file_mgr_->alloc(num_bytes);
 }

void PersistentStorageMgr::checkpoint ( )

override

Definition at line 158 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                       {
   global_file_mgr_->checkpoint();
 }

void PersistentStorageMgr::checkpoint	(	const int	db_id,
		const int	tb_id
	)

override

Definition at line 162 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                                                       {
   global_file_mgr_->checkpoint(db_id, tb_id);
 }

void PersistentStorageMgr::clearSlabs ( )

override

Definition at line 138 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                       {
   global_file_mgr_->clearSlabs();
 }

AbstractBuffer * PersistentStorageMgr::createBuffer	(	const ChunkKey &	chunk_key,
		const size_t	page_size,
		const size_t	initial_size
	)

override

Definition at line 53 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::createBuffer().

                                                                               {
   return getStorageMgrForTableKey(chunk_key)->createBuffer(
       chunk_key, page_size, initial_size);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

PersistentStorageMgr * PersistentStorageMgr::createPersistentStorageMgr	(	const std::string &	data_dir,
		const size_t	num_reader_threads,
		const DiskCacheConfig &	disk_cache_config
	)

static

Definition at line 23 of file PersistentStorageMgr.cpp.

References DiskCacheConfig::isEnabledForMutableTables(), and PersistentStorageMgr().

Referenced by Data_Namespace::DataMgr::populateMgrs().

                                    {
   if (config.isEnabledForMutableTables()) {
     return new MutableCachePersistentStorageMgr(data_dir, num_reader_threads, config);
   } else {
     return new PersistentStorageMgr(data_dir, num_reader_threads, config);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void PersistentStorageMgr::deleteBuffer	(	const ChunkKey &	chunk_key,
		const bool	purge
	)

override

Definition at line 60 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::deleteBuffer().

                                                                                    {
   getStorageMgrForTableKey(chunk_key)->deleteBuffer(chunk_key, purge);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void PersistentStorageMgr::deleteBuffersWithPrefix	(	const ChunkKey &	chunk_key_prefix,
		const bool	purge
	)

override

Definition at line 64 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::deleteBuffersWithPrefix().

                                                                      {
   getStorageMgrForTableKey(chunk_key_prefix)
       ->deleteBuffersWithPrefix(chunk_key_prefix, purge);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void PersistentStorageMgr::fetchBuffer	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	destination_buffer,
		const size_t	num_bytes
	)

override

Definition at line 75 of file PersistentStorageMgr.cpp.

References Data_Namespace::AbstractBuffer::copyTo(), disk_cache_, getStorageMgrForTableKey(), isChunkPrefixCacheable(), and isForeignStorage().

                                                                {
   AbstractBufferMgr* mgr = getStorageMgrForTableKey(chunk_key);
   if (isChunkPrefixCacheable(chunk_key)) {
     AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
     if (buffer) {
       buffer->copyTo(destination_buffer, num_bytes);
       return;
     } else {
       mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
       if (!isForeignStorage(chunk_key)) {
         // Foreign storage will read into cache buffers directly if enabled, so we do
         // not want to cache foreign table chunks here as they will already be cached.
         disk_cache_->cacheChunk(chunk_key, destination_buffer);
       }
       return;
     }
   }
   mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
 }

Here is the call graph for this function:

void PersistentStorageMgr::free ( AbstractBuffer * buffer )

override

Definition at line 170 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                                       {
   global_file_mgr_->free(buffer);
 }

size_t PersistentStorageMgr::getAllocated ( )

override

Definition at line 150 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                           {
   return global_file_mgr_->getAllocated();
 }

AbstractBuffer * PersistentStorageMgr::getBuffer	(	const ChunkKey &	chunk_key,
		const size_t	num_bytes
	)

override

Definition at line 70 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

                                                                         {
   return getStorageMgrForTableKey(chunk_key)->getBuffer(chunk_key, num_bytes);
 }

Here is the call graph for this function:

void PersistentStorageMgr::getChunkMetadataVecForKeyPrefix	(	ChunkMetadataVector &	chunk_metadata,
		const ChunkKey &	chunk_key_prefix
	)

override

Definition at line 104 of file PersistentStorageMgr.cpp.

References CHECK, disk_cache_, get_table_key(), getStorageMgrForTableKey(), has_table_prefix(), and isChunkPrefixCacheable().

                                {
   CHECK(has_table_prefix(keyPrefix));
   // If the disk has any cached metadata for a prefix then it is guaranteed to have all
   // metadata for that table, so we can return a complete set.  If it has no metadata,
   // then it may be that the table has no data, or that it's just not cached, so we need
   // to go to storage to check.
   if (isChunkPrefixCacheable(keyPrefix)) {
     if (disk_cache_->hasCachedMetadataForKeyPrefix(keyPrefix)) {
       disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, keyPrefix);
       return;
     } else {  // if we have no cached data attempt a recovery.
       if (disk_cache_->recoverCacheForTable(chunk_metadata, get_table_key(keyPrefix))) {
         return;
       }
     }
     getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
                                                                          keyPrefix);
     disk_cache_->cacheMetadataVec(chunk_metadata);
   } else {
     getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
                                                                          keyPrefix);
   }
 }

Here is the call graph for this function:

foreign_storage::ForeignStorageCache * PersistentStorageMgr::getDiskCache ( ) const

Definition at line 231 of file PersistentStorageMgr.cpp.

References disk_cache_.

                                                                              {
   return disk_cache_ ? disk_cache_.get() : nullptr;
 }

const DiskCacheConfig PersistentStorageMgr::getDiskCacheConfig ( ) const

inline

Definition at line 71 of file PersistentStorageMgr.h.

71 { return disk_cache_config_; }

PersistentStorageMgr::disk_cache_config_

DiskCacheConfig disk_cache_config_

Definition: PersistentStorageMgr.h:82

foreign_storage::ForeignStorageMgr * PersistentStorageMgr::getForeignStorageMgr ( ) const

Definition at line 227 of file PersistentStorageMgr.cpp.

References foreign_storage_mgr_.

                                                                                    {
   return foreign_storage_mgr_.get();
 }

File_Namespace::GlobalFileMgr * PersistentStorageMgr::getGlobalFileMgr ( ) const

Definition at line 186 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                                                           {
   return global_file_mgr_.get();
 }

size_t PersistentStorageMgr::getInUseSize ( )

override

Definition at line 146 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                           {
   return global_file_mgr_->getInUseSize();
 }

size_t PersistentStorageMgr::getMaxSize ( )

override

Definition at line 142 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                         {
   return global_file_mgr_->getMaxSize();
 }

MgrType PersistentStorageMgr::getMgrType ( )

override

Definition at line 174 of file PersistentStorageMgr.cpp.

                                          {
   return PERSISTENT_STORAGE_MGR;
 }

size_t PersistentStorageMgr::getNumChunks ( )

override

Definition at line 182 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                           {
   return global_file_mgr_->getNumChunks();
 }

AbstractBufferMgr * PersistentStorageMgr::getStorageMgrForTableKey ( const ChunkKey & table_key ) const

protected

Definition at line 218 of file PersistentStorageMgr.cpp.

References foreign_storage_mgr_, global_file_mgr_, and isForeignStorage().

Referenced by createBuffer(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), getBuffer(), getChunkMetadataVecForKeyPrefix(), putBuffer(), and removeTableRelatedDS().

                                      {
   if (isForeignStorage(table_key)) {
     return foreign_storage_mgr_.get();
   } else {
     return global_file_mgr_.get();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string PersistentStorageMgr::getStringMgrType ( )

override

Definition at line 178 of file PersistentStorageMgr.cpp.

                                                  {
   return ToString(PERSISTENT_STORAGE_MGR);
 }

bool PersistentStorageMgr::isAllocationCapped ( )

override

Definition at line 154 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                               {
   return global_file_mgr_->isAllocationCapped();
 }

bool PersistentStorageMgr::isBufferOnDevice ( const ChunkKey & chunk_key )

override

Definition at line 130 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                                                      {
   return global_file_mgr_->isBufferOnDevice(chunk_key);
 }

bool PersistentStorageMgr::isChunkPrefixCacheable ( const ChunkKey & chunk_prefix ) const

protected

Definition at line 235 of file PersistentStorageMgr.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, disk_cache_config_, has_table_prefix(), DiskCacheConfig::isEnabledForFSI(), DiskCacheConfig::isEnabledForMutableTables(), isForeignStorage(), and ForeignStorageInterface::lookupBufferManager().

Referenced by MutableCachePersistentStorageMgr::createBuffer(), fetchBuffer(), getChunkMetadataVecForKeyPrefix(), and removeTableRelatedDS().

                                                                                     {
   CHECK(has_table_prefix(chunk_prefix));
   // If this is an Arrow FSI table then we can't cache it.
   if (ForeignStorageInterface::lookupBufferManager(chunk_prefix[CHUNK_KEY_DB_IDX],
                                                    chunk_prefix[CHUNK_KEY_TABLE_IDX])) {
     return false;
   }
   return ((disk_cache_config_.isEnabledForMutableTables() &&
            !isForeignStorage(chunk_prefix)) ||
           (disk_cache_config_.isEnabledForFSI() && isForeignStorage(chunk_prefix)));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool PersistentStorageMgr::isForeignStorage ( const ChunkKey & chunk_key ) const

protected

Definition at line 198 of file PersistentStorageMgr.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, StorageType::FOREIGN_TABLE, Catalog_Namespace::SysCatalog::getCatalog(), has_table_prefix(), and Catalog_Namespace::SysCatalog::instance().

Referenced by MutableCachePersistentStorageMgr::deleteBuffer(), fetchBuffer(), getStorageMgrForTableKey(), and isChunkPrefixCacheable().

                                                                            {
   CHECK(has_table_prefix(chunk_key));
   auto db_id = chunk_key[CHUNK_KEY_DB_IDX];
   auto table_id = chunk_key[CHUNK_KEY_TABLE_IDX];
   auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
 
   // if catalog doesnt exist at this point we must be in an old migration.
   // Old migration can not, at this point 5.5.1, be using foreign storage
   // so this hack is to avoid the crash, when migrating old
   // catalogs that have not been upgraded over time due to issue
   // [BE-5728]
   if (!catalog) {
     return false;
   }
 
   auto table = catalog->getMetadataForTableImpl(table_id, false);
   CHECK(table);
   return table->storageType == StorageType::FOREIGN_TABLE;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string PersistentStorageMgr::printSlabs ( )

override

Definition at line 134 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

                                            {
   return global_file_mgr_->printSlabs();
 }

AbstractBuffer * PersistentStorageMgr::putBuffer	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	source_buffer,
		const size_t	num_bytes
	)

override

Definition at line 97 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::putBuffer().

                                                                         {
   return getStorageMgrForTableKey(chunk_key)->putBuffer(
       chunk_key, source_buffer, num_bytes);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

int PersistentStorageMgr::recoverDataWrapperIfCachedAndGetHighestFragId ( const ChunkKey & table_key )

protected

void PersistentStorageMgr::removeTableRelatedDS	(	const int	db_id,
		const int	table_id
	)

override

Definition at line 190 of file PersistentStorageMgr.cpp.

References disk_cache_, getStorageMgrForTableKey(), and isChunkPrefixCacheable().

Referenced by MutableCachePersistentStorageMgr::removeTableRelatedDS().

                                                                                    {
   const ChunkKey table_key{db_id, table_id};
   if (isChunkPrefixCacheable(table_key)) {
     disk_cache_->clearForTablePrefix(table_key);
   }
   getStorageMgrForTableKey(table_key)->removeTableRelatedDS(db_id, table_id);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

std::unique_ptr<foreign_storage::ForeignStorageCache> PersistentStorageMgr::disk_cache_

protected

Definition at line 81 of file PersistentStorageMgr.h.

Referenced by MutableCachePersistentStorageMgr::checkpoint(), MutableCachePersistentStorageMgr::deleteBuffer(), MutableCachePersistentStorageMgr::deleteBuffersWithPrefix(), fetchBuffer(), getChunkMetadataVecForKeyPrefix(), getDiskCache(), MutableCachePersistentStorageMgr::MutableCachePersistentStorageMgr(), PersistentStorageMgr(), MutableCachePersistentStorageMgr::putBuffer(), and removeTableRelatedDS().

DiskCacheConfig PersistentStorageMgr::disk_cache_config_

protected

Definition at line 82 of file PersistentStorageMgr.h.

Referenced by isChunkPrefixCacheable(), MutableCachePersistentStorageMgr::MutableCachePersistentStorageMgr(), and PersistentStorageMgr().

std::unique_ptr<foreign_storage::ForeignStorageMgr> PersistentStorageMgr::foreign_storage_mgr_

protected

Definition at line 80 of file PersistentStorageMgr.h.

Referenced by getForeignStorageMgr(), getStorageMgrForTableKey(), and PersistentStorageMgr().

std::unique_ptr<File_Namespace::GlobalFileMgr> PersistentStorageMgr::global_file_mgr_

protected

Definition at line 79 of file PersistentStorageMgr.h.

Referenced by alloc(), MutableCachePersistentStorageMgr::checkpoint(), checkpoint(), clearSlabs(), free(), getAllocated(), getGlobalFileMgr(), getInUseSize(), getMaxSize(), getNumChunks(), getStorageMgrForTableKey(), isAllocationCapped(), isBufferOnDevice(), and printSlabs().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/PersistentStorageMgr/PersistentStorageMgr.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/PersistentStorageMgr/PersistentStorageMgr.cpp

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation