#include <ForeignStorageCache.h>

Public Member Functions
	ForeignStorageCache (const File_Namespace::DiskCacheConfig &config)

void	checkpoint (const int32_t db_id, const int32_t tb_id)

void	putBuffer (const ChunkKey &, AbstractBuffer *, const size_t numBytes=0)

File_Namespace::FileBuffer *	getCachedChunkIfExists (const ChunkKey &)

bool	isMetadataCached (const ChunkKey &) const

void	cacheMetadataVec (const ChunkMetadataVector &)

void	getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const

bool	hasCachedMetadataForKeyPrefix (const ChunkKey &) const

void	clearForTablePrefix (const ChunkKey &)

void	clear ()

size_t	getMaxChunkDataSize () const

std::vector< ChunkKey >	getCachedChunksForKeyPrefix (const ChunkKey &) const

ChunkToBufferMap	getChunkBuffersForCaching (const std::set< ChunkKey > &chunk_keys) const

AbstractBuffer *	getChunkBufferForPrecaching (const ChunkKey &chunk_key, bool is_new_buffer)

void	deleteBufferIfExists (const ChunkKey &chunk_key)

size_t	getNumCachedChunks () const

size_t	getNumCachedMetadata () const

std::string	dumpCachedChunkEntries () const

std::string	dumpCachedMetadataEntries () const

std::string	dumpEvictionQueue () const

std::string	dump () const

std::string	getCacheDirectory () const

std::string	getCacheDirectoryForTable (int db_id, int tb_id) const

std::string	getSerializedWrapperPath (int32_t db_id, int32_t tb_id) const

uint64_t	getSpaceReservedByTable (int db_id, int tb_id) const

void	storeDataWrapper (const std::string &doc, int32_t db_id, int32_t tb_id)

bool	hasStoredDataWrapperMetadata (int32_t db_id, int32_t table_id) const

void	eraseChunk (const ChunkKey &chunk_key)

void	setDataSizeLimit (size_t max) const

Private Member Functions
void	validatePath (const std::string &) const

Private Attributes
std::unique_ptr < File_Namespace::CachingFileMgr >	caching_file_mgr_

Detailed Description

Definition at line 42 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache ( const File_Namespace::DiskCacheConfig & config )

Definition at line 52 of file ForeignStorageCache.cpp.

References caching_file_mgr_, File_Namespace::DiskCacheConfig::path, and validatePath().

                                                                                     {
   validatePath(config.path);
   caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(config);
 }

Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheMetadataVec ( const ChunkMetadataVector & metadata_vec )

Definition at line 109 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, DEBUG_TIMER, eraseChunk(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), in_same_table(), is_varlen_data_key(), is_varlen_key(), foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix(), foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

                                                                                   {
   auto timer = DEBUG_TIMER(__func__);
   if (metadata_vec.empty()) {
     return;
   }
   auto first_chunk_key = metadata_vec.begin()->first;
   for (auto& [chunk_key, metadata] : metadata_vec) {
     CHECK(in_same_table(chunk_key, first_chunk_key));
     AbstractBuffer* buf;
     AbstractBuffer* index_buffer = nullptr;
     ChunkKey index_chunk_key;
     if (is_varlen_key(chunk_key)) {
       // For variable length chunks, metadata is associated with the data chunk.
       CHECK(is_varlen_data_key(chunk_key));
       index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
                          chunk_key[CHUNK_KEY_TABLE_IDX],
                          chunk_key[CHUNK_KEY_COLUMN_IDX],
                          chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                          2};
     }
     bool chunk_in_cache = false;
     if (!caching_file_mgr_->isBufferOnDevice(chunk_key)) {
       buf = caching_file_mgr_->createBuffer(chunk_key);
 
       if (!index_chunk_key.empty()) {
         CHECK(!caching_file_mgr_->isBufferOnDevice(index_chunk_key));
         index_buffer = caching_file_mgr_->createBuffer(index_chunk_key);
         CHECK(index_buffer);
       }
     } else {
       buf = caching_file_mgr_->getBuffer(chunk_key);
 
       if (!index_chunk_key.empty()) {
         CHECK(caching_file_mgr_->isBufferOnDevice(index_chunk_key));
         index_buffer = caching_file_mgr_->getBuffer(index_chunk_key);
         CHECK(index_buffer);
       }
 
       // We should have already cleared the data unless we are appending
       // If the buffer metadata has changed, we need to remove this chunk
       if (buf->getEncoder() != nullptr) {
         const std::shared_ptr<ChunkMetadata> buf_metadata =
             std::make_shared<ChunkMetadata>();
         buf->getEncoder()->getMetadata(buf_metadata);
         chunk_in_cache = *metadata.get() == *buf_metadata;
       }
     }
 
     if (!chunk_in_cache) {
       set_metadata_for_buffer(buf, metadata.get());
       eraseChunk(chunk_key);
 
       if (!index_chunk_key.empty()) {
         CHECK(index_buffer);
         index_buffer->setUpdated();
         eraseChunk(index_chunk_key);
       }
     }
   }
   caching_file_mgr_->checkpoint(first_chunk_key[CHUNK_KEY_DB_IDX],
                                 first_chunk_key[CHUNK_KEY_TABLE_IDX]);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::checkpoint	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 68 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper(), and foreign_storage::CachingForeignStorageMgr::populateChunkBuffersSafely().

                                                                              {
   caching_file_mgr_->checkpoint(db_id, tb_id);
 }

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear ( )

Definition at line 190 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and DEBUG_TIMER.

                                 {
   auto timer = DEBUG_TIMER(__func__);
   // FileMgrs do not clean up after themselves nicely, so we need to close all their disk
   // resources and then re-create the CachingFileMgr to reset it.
   caching_file_mgr_->closeRemovePhysical();
   boost::filesystem::create_directory(caching_file_mgr_->getFileMgrBasePath());
   caching_file_mgr_ = caching_file_mgr_->reconstruct();
 }

void foreign_storage::ForeignStorageCache::clearForTablePrefix ( const ChunkKey & chunk_prefix )

Definition at line 183 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, DEBUG_TIMER, and is_table_key().

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::clear_cached_table_data(), foreign_storage::CachingForeignStorageMgr::clearTable(), File_Namespace::CachingGlobalFileMgr::deleteBuffersWithPrefix(), foreign_storage::CachingForeignStorageMgr::removeTableRelatedDS(), and File_Namespace::CachingGlobalFileMgr::removeTableRelatedDS().

                                                                           {
   CHECK(is_table_key(chunk_prefix));
   auto timer = DEBUG_TIMER(__func__);
   caching_file_mgr_->clearForTable(chunk_prefix[CHUNK_KEY_DB_IDX],
                                    chunk_prefix[CHUNK_KEY_TABLE_IDX]);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists ( const ChunkKey & chunk_key )

Definition at line 57 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffer().

                                                                         {
   caching_file_mgr_->deleteBufferIfExists(chunk_key);
 }

Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::dump ( ) const

inline

Definition at line 77 of file ForeignStorageCache.h.

77 { return caching_file_mgr_->dump(); }

foreign_storage::ForeignStorageCache::caching_file_mgr_

std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

Definition: ForeignStorageCache.h:111

std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries ( ) const

Definition at line 208 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

                                                             {
   return caching_file_mgr_->dumpKeysWithChunkData();
 }

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries ( ) const

Definition at line 212 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

                                                                {
   return caching_file_mgr_->dumpKeysWithMetadata();
 }

std::string foreign_storage::ForeignStorageCache::dumpEvictionQueue ( ) const

Definition at line 216 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

                                                        {
   return caching_file_mgr_->dumpEvictionQueue();
 }

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey & chunk_key )

Definition at line 204 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by cacheMetadataVec(), foreign_storage::CachingForeignStorageMgr::evictChunkFromCache(), and foreign_storage::CachingForeignStorageMgr::fetchBuffer().

                                                               {
   caching_file_mgr_->removeChunkKeepMetadata(chunk_key);
 }

Here is the caller graph for this function:

File_Namespace::FileBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists ( const ChunkKey & chunk_key )

Definition at line 72 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHUNK_KEY_VARLEN_IDX, and is_varlen_data_key().

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), foreign_storage::CachingForeignStorageMgr::isChunkCached(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                {
   auto buf = caching_file_mgr_->getBufferIfExists(chunk_key);
 
   if (buf) {
     if ((*buf)->hasDataPages()) {
       // 1. If the buffer has data pages then must be in the cache.
       return *buf;
     }
     if (is_varlen_data_key(chunk_key)) {
       // 2. If the buffer is a varlen data buffer and the
       // corresponding chunk contains only nulls, then even
       // without data pages it will still have been cached
       // if it has a corresponding index buffer which does
       // have dataPages
       // Note the empty buffer proviso that the size be 0,
       // corresponding to  all nulls in the chunk
       auto index_chunk_key = chunk_key;
       index_chunk_key[CHUNK_KEY_VARLEN_IDX] = 2;
       auto index_buffer = caching_file_mgr_->getBufferIfExists(index_chunk_key);
       if (index_buffer && (*index_buffer)->hasDataPages() && (*buf)->size() == 0) {
         return *buf;
       }
     }
   }
   // 3. Otherwise this chunk hasn't been cached.
   return nullptr;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix ( const ChunkKey & chunk_prefix ) const

Definition at line 199 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                         {
   return caching_file_mgr_->getChunkKeysForPrefix(chunk_prefix);
 }

Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectory ( ) const

inline

Definition at line 79 of file ForeignStorageCache.h.

                                              {
     return caching_file_mgr_->getFileMgrBasePath();
   }

std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTable	(	int	db_id,
		int	tb_id
	)		const

inline

Definition at line 83 of file ForeignStorageCache.h.

                                                                          {
     return caching_file_mgr_->getTableFileMgrPath(db_id, tb_id);
   }

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix	(	ChunkMetadataVector &	metadata_vec,
		const ChunkKey &	chunk_prefix
	)		const

Definition at line 172 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::createDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getBufferSize(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

                                         {
   caching_file_mgr_->getChunkMetadataVecForKeyPrefix(metadata_vec, chunk_prefix);
 }

Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getChunkBufferForPrecaching	(	const ChunkKey &	chunk_key,
		bool	is_new_buffer
	)

Definition at line 256 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and CHECK.

                         {
   if (!is_new_buffer) {
     CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
     return caching_file_mgr_->getBuffer(chunk_key);
   } else {
     CHECK(!caching_file_mgr_->isBufferOnDevice(chunk_key));
     return caching_file_mgr_->createBuffer(chunk_key);
   }
 }

ChunkToBufferMap foreign_storage::ForeignStorageCache::getChunkBuffersForCaching ( const std::set< ChunkKey > & chunk_keys ) const

Definition at line 240 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and CHECK.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                         {
   ChunkToBufferMap chunk_buffer_map;
   for (const auto& key : keys) {
     CHECK(caching_file_mgr_->isBufferOnDevice(key));
     chunk_buffer_map[key] = caching_file_mgr_->getBuffer(key);
     auto file_buf = dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[key]);
     CHECK(file_buf);
     CHECK(!file_buf->hasDataPages());
 
     // Clear all buffer metadata
     file_buf->resetToEmpty();
   }
   return chunk_buffer_map;
 }

Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getMaxChunkDataSize ( ) const

inline

Definition at line 55 of file ForeignStorageCache.h.

Referenced by foreign_storage::CachingForeignStorageMgr::maxFetchSize().

55 { return caching_file_mgr_->getMaxDataFilesSize(); }

foreign_storage::ForeignStorageCache::caching_file_mgr_

std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

Definition: ForeignStorageCache.h:111

Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const

inline

Definition at line 66 of file ForeignStorageCache.h.

                                            {
     return caching_file_mgr_->getNumDataChunks();
   }

size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const

inline

Definition at line 69 of file ForeignStorageCache.h.

                                              {
     return caching_file_mgr_->getNumChunksWithMetadata();
   }

std::string foreign_storage::ForeignStorageCache::getSerializedWrapperPath	(	int32_t	db_id,
		int32_t	tb_id
	)		const

inline

Definition at line 87 of file ForeignStorageCache.h.

References File_Namespace::CachingFileMgr::WRAPPER_FILE_NAME.

Referenced by foreign_storage::CachingForeignStorageMgr::createDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::eraseDataWrapper().

                                                                                 {
     return getCacheDirectoryForTable(db_id, tb_id) + "/" +
            File_Namespace::CachingFileMgr::WRAPPER_FILE_NAME;
   }

Here is the caller graph for this function:

uint64_t foreign_storage::ForeignStorageCache::getSpaceReservedByTable	(	int	db_id,
		int	tb_id
	)		const

inline

Definition at line 92 of file ForeignStorageCache.h.

                                                                       {
     return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
   }

bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix ( const ChunkKey & chunk_prefix ) const

Definition at line 178 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::clearTable(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

                                         {
   return caching_file_mgr_->hasChunkMetadataForKeyPrefix(chunk_prefix);
 }

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::hasStoredDataWrapperMetadata	(	int32_t	db_id,
		int32_t	table_id
	)		const

Definition at line 274 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::hasStoredDataWrapper().

                                                                                {
   return caching_file_mgr_->hasWrapperFile(db_id, table_id);
 }

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached ( const ChunkKey & chunk_key ) const

Definition at line 101 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                           {
   auto buf = caching_file_mgr_->getBufferIfExists(chunk_key);
   if (buf) {
     return (*buf)->hasEncoder();
   }
   return false;
 }

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::putBuffer	(	const ChunkKey &	key,
		AbstractBuffer *	buf,
		const size_t	numBytes = `0`
	)

Definition at line 61 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHECK, and Data_Namespace::AbstractBuffer::isDirty().

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), and File_Namespace::CachingGlobalFileMgr::putBuffer().

                                                             {
   caching_file_mgr_->putBuffer(key, buf, num_bytes);
   CHECK(!buf->isDirty());
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::setDataSizeLimit ( size_t max ) const

inline

Definition at line 103 of file ForeignStorageCache.h.

                                                  {
     caching_file_mgr_->setDataSizeLimit(max);
   }

void foreign_storage::ForeignStorageCache::storeDataWrapper	(	const std::string &	doc,
		int32_t	db_id,
		int32_t	tb_id
	)

Definition at line 268 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper().

                                                           {
   caching_file_mgr_->writeWrapperFile(doc, db_id, tb_id);
 }

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string & base_path ) const

private

Definition at line 220 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

                                                                        {
   // check if base_path already exists, and if not create one
   boost::filesystem::path path(base_path);
   if (boost::filesystem::exists(path)) {
     if (!boost::filesystem::is_directory(path)) {
       throw std::runtime_error{
           "cache path \"" + base_path +
           "\" is not a directory.  Please specify a valid directory "
           "with --disk_cache_path=<path>, or use the default location."};
     }
   } else {  // data directory does not exist
     if (!boost::filesystem::create_directory(path)) {
       throw std::runtime_error{
           "could not create directory at cache path \"" + base_path +
           "\".  Please specify a valid directory location "
           "with --disk_cache_path=<path> or use the default location."};
     }
   }
 }

Here is the caller graph for this function:

Member Data Documentation

std::unique_ptr<File_Namespace::CachingFileMgr> foreign_storage::ForeignStorageCache::caching_file_mgr_

private

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.cpp

Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation