#include <CachingForeignStorageMgr.h>

Inheritance diagram for foreign_storage::CachingForeignStorageMgr:

Collaboration diagram for foreign_storage::CachingForeignStorageMgr:

Public Member Functions
	CachingForeignStorageMgr (ForeignStorageCache *cache)

void	fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override

void	getChunkMetadataVecFromDataWrapper (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix)

void	removeTableRelatedDS (const int db_id, const int table_id) override

void	refreshTable (const ChunkKey &table_key, const bool evict_cached_entries) override

bool	createDataWrapperIfNotExists (const ChunkKey &chunk_key) override

bool	hasStoredDataWrapper (int32_t db, int32_t tb) const

Public Member Functions inherited from foreign_storage::ForeignStorageMgr
	ForeignStorageMgr ()

	~ForeignStorageMgr () override

AbstractBuffer *	createBuffer (const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override

void	deleteBuffer (const ChunkKey &chunk_key, const bool purge) override

void	deleteBuffersWithPrefix (const ChunkKey &chunk_key_prefix, const bool purge) override

AbstractBuffer *	getBuffer (const ChunkKey &chunk_key, const size_t num_bytes) override

void	fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override

AbstractBuffer *	putBuffer (const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override

bool	isBufferOnDevice (const ChunkKey &chunk_key) override

std::string	printSlabs () override

size_t	getMaxSize () override

size_t	getInUseSize () override

size_t	getAllocated () override

bool	isAllocationCapped () override

void	checkpoint () override

void	checkpoint (const int db_id, const int tb_id) override

AbstractBuffer *	alloc (const size_t num_bytes) override

void	free (AbstractBuffer *buffer) override

MgrType	getMgrType () override

std::string	getStringMgrType () override

size_t	getNumChunks () override

void	removeTableRelatedDS (const int db_id, const int table_id) override

bool	hasDataWrapperForChunk (const ChunkKey &chunk_key) const

bool	isDatawrapperRestored (const ChunkKey &chunk_key)

void	setDataWrapper (const ChunkKey &table_key, std::shared_ptr< MockForeignDataWrapper > data_wrapper)

std::shared_ptr < ForeignDataWrapper >	getDataWrapper (const ChunkKey &chunk_key) const

void	setParallelismHints (const std::map< ChunkKey, std::set< ParallelismHint >> &hints_per_table)

Private Member Functions
void	refreshTableInCache (const ChunkKey &table_key)

int	getHighestCachedFragId (const ChunkKey &table_key)

void	refreshAppendTableInCache (const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)

void	refreshNonAppendTableInCache (const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)

void	refreshChunksInCacheByFragment (const std::vector< ChunkKey > &old_chunk_keys, int last_frag_id)

void	populateChunkBuffersSafely (ForeignDataWrapper &data_wrapper, ChunkToBufferMap &required_buffers, ChunkToBufferMap &optional_buffers)

void	eraseDataWrapper (const ChunkKey &key) override

void	clearTable (const ChunkKey &table_key)

size_t	maxFetchSize (int32_t db_id) const override

bool	hasMaxFetchSize () const override

std::set< ChunkKey >	getOptionalKeysWithinSizeLimit (const ChunkKey &chunk_key, const std::set< ChunkKey, decltype(set_comp)* > &same_fragment_keys, const std::set< ChunkKey, decltype(set_comp)* > &diff_fragment_keys) const override

size_t	getBufferSize (const ChunkKey &key) const

size_t	getRequiredBuffersSize (const ChunkKey &chunk_key) const

Private Attributes
ForeignStorageCache *	disk_cache_

Additional Inherited Members
Public Types inherited from foreign_storage::ForeignStorageMgr
using	ParallelismHint = std::pair< int, int >

Protected Member Functions inherited from foreign_storage::ForeignStorageMgr
void	updateFragmenterMetadata (const ChunkToBufferMap &) const

void	createDataWrapperUnlocked (int32_t db, int32_t tb)

bool	fetchBufferIfTempBufferMapEntryExists (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes)

ChunkToBufferMap	allocateTempBuffersForChunks (const std::set< ChunkKey > &chunk_keys)

void	clearTempChunkBufferMapEntriesForTable (const ChunkKey &table_key)

void	clearTempChunkBufferMapEntriesForTableUnlocked (const ChunkKey &table_key)

std::set< ChunkKey >	getOptionalChunkKeySet (const ChunkKey &chunk_key, const std::set< ChunkKey > &required_chunk_keys, const ForeignDataWrapper::ParallelismLevel parallelism_level) const

std::pair< std::set< ChunkKey, decltype(set_comp)* > , std::set< ChunkKey, decltype(set_comp)* > >	getPrefetchSets (const ChunkKey &chunk_key, const std::set< ChunkKey > &required_chunk_keys, const ForeignDataWrapper::ParallelismLevel parallelism_level) const

Static Protected Member Functions inherited from foreign_storage::ForeignStorageMgr
static void	checkIfS3NeedsToBeEnabled (const ChunkKey &chunk_key)

Protected Attributes inherited from foreign_storage::ForeignStorageMgr
std::shared_mutex	data_wrapper_mutex_

std::map< ChunkKey, std::shared_ptr < ForeignDataWrapper > >	data_wrapper_map_

std::map< ChunkKey, std::shared_ptr < MockForeignDataWrapper > >	mocked_wrapper_map_

std::map< ChunkKey, std::unique_ptr < AbstractBuffer > >	temp_chunk_buffer_map_

std::shared_mutex	temp_chunk_buffer_map_mutex_

std::shared_mutex	parallelism_hints_mutex_

std::map< ChunkKey, std::set < ParallelismHint > >	parallelism_hints_per_table_

Detailed Description

Definition at line 27 of file CachingForeignStorageMgr.h.

Constructor & Destructor Documentation

foreign_storage::CachingForeignStorageMgr::CachingForeignStorageMgr ( ForeignStorageCache * cache )

Definition at line 37 of file CachingForeignStorageMgr.cpp.

References CHECK, and disk_cache_.

     : ForeignStorageMgr(), disk_cache_(cache) {
   CHECK(disk_cache_);
 }

Member Function Documentation

void foreign_storage::CachingForeignStorageMgr::clearTable ( const ChunkKey & table_key )

private

Definition at line 253 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageCache::clearForTablePrefix(), disk_cache_, foreign_storage::ForeignStorageMgr::eraseDataWrapper(), and foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix().

Referenced by getChunkMetadataVecFromDataWrapper(), refreshNonAppendTableInCache(), and refreshTable().

                                                                    {
   disk_cache_->clearForTablePrefix(table_key);
   CHECK(!disk_cache_->hasCachedMetadataForKeyPrefix(table_key));
   ForeignStorageMgr::eraseDataWrapper(table_key);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::CachingForeignStorageMgr::createDataWrapperIfNotExists ( const ChunkKey & chunk_key )

overridevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 375 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageMgr::createDataWrapperUnlocked(), foreign_storage::ForeignStorageMgr::data_wrapper_map_, foreign_storage::ForeignStorageMgr::data_wrapper_mutex_, disk_cache_, get_table_key(), get_table_prefix(), foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), and foreign_storage::ForeignStorageCache::getSerializedWrapperPath().

Referenced by getChunkMetadataVecForKeyPrefix().

                                                                                      {
   std::lock_guard data_wrapper_lock(data_wrapper_mutex_);
   ChunkKey table_key = get_table_key(chunk_key);
   auto data_wrapper_it = data_wrapper_map_.find(table_key);
   if (data_wrapper_it != data_wrapper_map_.end()) {
     return false;
   }
   auto [db, tb] = get_table_prefix(chunk_key);
   createDataWrapperUnlocked(db, tb);
   auto wrapper_file = disk_cache_->getSerializedWrapperPath(db, tb);
   if (boost::filesystem::exists(wrapper_file)) {
     ChunkMetadataVector chunk_metadata;
     disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, table_key);
     data_wrapper_map_.at(table_key)->restoreDataWrapperInternals(
         disk_cache_->getSerializedWrapperPath(db, tb), chunk_metadata);
   }
   return true;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::eraseDataWrapper ( const ChunkKey & key )

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 240 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageMgr::data_wrapper_map_, foreign_storage::ForeignStorageMgr::data_wrapper_mutex_, disk_cache_, get_table_prefix(), foreign_storage::ForeignStorageCache::getSerializedWrapperPath(), and is_table_key().

Referenced by getChunkMetadataVecForKeyPrefix().

                                                                    {
   CHECK(is_table_key(key));
   std::lock_guard data_wrapper_lock(data_wrapper_mutex_);
   // May not be created yet
   if (data_wrapper_map_.find(key) != data_wrapper_map_.end()) {
     auto [db, tb] = get_table_prefix(key);
     // Need to erase serialized version on disk if it exists so we don't accidentally
     // recover it after deleting.
     boost::filesystem::remove_all(disk_cache_->getSerializedWrapperPath(db, tb));
     data_wrapper_map_.erase(key);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::fetchBuffer	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	destination_buffer,
		const size_t	num_bytes
	)

override

Definition at line 82 of file CachingForeignStorageMgr.cpp.

References CHECK, CHUNK_KEY_DB_IDX, Data_Namespace::AbstractBuffer::copyTo(), disk_cache_, foreign_storage::ForeignStorageMgr::fetchBuffer(), foreign_storage::get_column_key_set(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), foreign_storage::ForeignStorageCache::getChunkBuffersForCaching(), foreign_storage::ForeignStorageMgr::getDataWrapper(), foreign_storage::ForeignStorageMgr::getOptionalChunkKeySet(), getRequiredBuffersSize(), foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::is_in_memory_system_table_chunk_key(), Data_Namespace::AbstractBuffer::isDirty(), maxFetchSize(), populateChunkBuffersSafely(), and foreign_storage::ChunkSizeValidator::validateChunkSize().

                                                                    {
   ChunkSizeValidator chunk_size_validator(chunk_key);
   if (is_in_memory_system_table_chunk_key(chunk_key)) {
     ForeignStorageMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
     chunk_size_validator.validateChunkSize(destination_buffer);
     return;
   }
   CHECK(destination_buffer);
   CHECK(!destination_buffer->isDirty());
 
   AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
   if (buffer) {
     chunk_size_validator.validateChunkSize(buffer);
     buffer->copyTo(destination_buffer, num_bytes);
     return;
   } else {
     auto required_size = getRequiredBuffersSize(chunk_key);
     if (required_size > maxFetchSize(chunk_key[CHUNK_KEY_DB_IDX])) {
       // If we don't have space in the cache then skip the caching.
       ForeignStorageMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
       return;
     }
 
     auto column_keys = get_column_key_set(chunk_key);
 
     // Use hints to prefetch other chunks in fragment into cache
     auto& data_wrapper = *getDataWrapper(chunk_key);
     auto optional_set = getOptionalChunkKeySet(
         chunk_key, column_keys, data_wrapper.getCachedParallelismLevel());
 
     // Remove any chunks that are already cached.
     // TODO(Misiu): Change to use std::erase_if when we get c++20
     for (auto it = optional_set.begin(); it != optional_set.end();) {
       if (disk_cache_->getCachedChunkIfExists(*it) != nullptr) {
         it = optional_set.erase(it);
       } else {
         ++it;
       }
     }
 
     auto optional_buffers = disk_cache_->getChunkBuffersForCaching(optional_set);
     auto required_buffers = disk_cache_->getChunkBuffersForCaching(column_keys);
     CHECK(required_buffers.find(chunk_key) != required_buffers.end());
     populateChunkBuffersSafely(data_wrapper, required_buffers, optional_buffers);
 
     AbstractBuffer* buffer = required_buffers.at(chunk_key);
     CHECK(buffer);
     buffer->copyTo(destination_buffer, num_bytes);
   }
 }

Here is the call graph for this function:

size_t foreign_storage::CachingForeignStorageMgr::getBufferSize ( const ChunkKey & key ) const

private

Definition at line 443 of file CachingForeignStorageMgr.cpp.

References CHECK_EQ, disk_cache_, get_fragment_key(), foreign_storage::get_max_chunk_size(), foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), is_varlen_data_key(), is_varlen_key(), and show_chunk().

Referenced by getOptionalKeysWithinSizeLimit(), and getRequiredBuffersSize().

                                                                         {
   size_t num_bytes = 0;
   ChunkMetadataVector meta;
   disk_cache_->getCachedMetadataVecForKeyPrefix(meta, get_fragment_key(key));
   CHECK_EQ(meta.size(), 1U) << show_chunk(key);
   auto metadata = meta.begin()->second;
 
   if (is_varlen_key(key)) {
     if (is_varlen_data_key(key)) {
       num_bytes = get_max_chunk_size(key);
     } else {
       num_bytes = (metadata->sqlType.is_string())
                       ? sizeof(StringOffsetT) * (metadata->numElements + 1)
                       : sizeof(ArrayOffsetT) * (metadata->numElements + 1);
     }
   } else {
     num_bytes = metadata->numBytes;
   }
   return num_bytes;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix	(	ChunkMetadataVector &	chunk_metadata,
		const ChunkKey &	chunk_key_prefix
	)

override

Definition at line 135 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, createDataWrapperIfNotExists(), disk_cache_, eraseDataWrapper(), foreign_storage::fragment_maps_to_leaf(), get_table_key(), foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), foreign_storage::ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), getChunkMetadataVecFromDataWrapper(), has_table_prefix(), foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix(), foreign_storage::ForeignStorageCache::hasStoredDataWrapperMetadata(), dist::is_distributed(), foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::is_in_memory_system_table_chunk_key(), foreign_storage::is_shardable_key(), and show_chunk().

                                 {
   if (is_in_memory_system_table_chunk_key(key_prefix)) {
     ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
     return;
   }
   CHECK(has_table_prefix(key_prefix));
   // If the disk has any cached metadata for a prefix then it is guaranteed to have all
   // metadata for that table, so we can return a complete set.  If it has no metadata,
   // then it may be that the table has no data, or that it's just not cached, so we need
   // to go to storage to check.
   if (disk_cache_->hasCachedMetadataForKeyPrefix(key_prefix)) {
     disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
 
     // Assert all metadata in cache is mapped to this leaf node in distributed.
     if (is_shardable_key(key_prefix)) {
       for (auto& [key, meta] : chunk_metadata) {
         CHECK(fragment_maps_to_leaf(key)) << show_chunk(key);
       }
     }
 
     // If the data in cache was restored from disk then it is possible that the wrapper
     // does not exist yet.  In this case the wrapper will be restored from disk if
     // possible.
     createDataWrapperIfNotExists(key_prefix);
     return;
   } else if (dist::is_distributed() &&
              disk_cache_->hasStoredDataWrapperMetadata(key_prefix[CHUNK_KEY_DB_IDX],
                                                        key_prefix[CHUNK_KEY_TABLE_IDX])) {
     // In distributed mode, it is possible to have all the chunk metadata filtered out for
     // this node, after previously getting the chunk metadata from the wrapper and caching
     // the wrapper metadata. In this case, return immediately and avoid doing a redundant
     // metadata scan.
     return;
   }
 
   // If we have no cached data then either the data was evicted, was never populated, or
   // the data for the table is an empty set (no chunks).  In case we are hitting the first
   // two, we should repopulate the data wrapper so just do it in all cases.
   auto table_key = get_table_key(key_prefix);
   eraseDataWrapper(table_key);
   createDataWrapperIfNotExists(table_key);
 
   getChunkMetadataVecFromDataWrapper(chunk_metadata, key_prefix);
   disk_cache_->cacheMetadataVec(chunk_metadata);
 }

Here is the call graph for this function:

void foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper	(	ChunkMetadataVector &	chunk_metadata,
		const ChunkKey &	chunk_key_prefix
	)

Definition at line 183 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageCache::checkpoint(), clearTable(), disk_cache_, get_table_prefix(), foreign_storage::ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), foreign_storage::ForeignStorageMgr::getDataWrapper(), has_table_prefix(), foreign_storage::is_table_enabled_on_node(), and foreign_storage::ForeignStorageCache::storeDataWrapper().

Referenced by getChunkMetadataVecForKeyPrefix(), refreshAppendTableInCache(), and refreshNonAppendTableInCache().

                                       {
   CHECK(has_table_prefix(chunk_key_prefix));
   auto [db_id, tb_id] = get_table_prefix(chunk_key_prefix);
   try {
     ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(chunk_metadata, chunk_key_prefix);
   } catch (...) {
     clearTable({db_id, tb_id});
     throw;
   }
   // If the table was disabled then we will have no wrapper to serialize.
   if (is_table_enabled_on_node(chunk_key_prefix)) {
     auto doc = getDataWrapper(chunk_key_prefix)->getSerializedDataWrapper();
     disk_cache_->storeDataWrapper(doc, db_id, tb_id);
 
     // If the wrapper populated buffers we want that action to be checkpointed.
     disk_cache_->checkpoint(db_id, tb_id);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

int foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId ( const ChunkKey & table_key )

private

Definition at line 259 of file CachingForeignStorageMgr.cpp.

References CHUNK_KEY_FRAGMENT_IDX, disk_cache_, foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), and foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix().

Referenced by refreshAppendTableInCache().

                                                                               {
   // Determine last fragment ID
   int last_frag_id = 0;
   if (disk_cache_->hasCachedMetadataForKeyPrefix(table_key)) {
     ChunkMetadataVector cached_metadata;
     disk_cache_->getCachedMetadataVecForKeyPrefix(cached_metadata, table_key);
     for (const auto& [key, metadata] : cached_metadata) {
       last_frag_id = std::max(last_frag_id, key[CHUNK_KEY_FRAGMENT_IDX]);
     }
   }
   return last_frag_id;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::set< ChunkKey > foreign_storage::CachingForeignStorageMgr::getOptionalKeysWithinSizeLimit	(	const ChunkKey &	chunk_key,
		const std::set< ChunkKey, decltype(set_comp)* > &	same_fragment_keys,
		const std::set< ChunkKey, decltype(set_comp)* > &	diff_fragment_keys
	)		const

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 416 of file CachingForeignStorageMgr.cpp.

References CHUNK_KEY_DB_IDX, foreign_storage::get_column_key_set(), getBufferSize(), getRequiredBuffersSize(), and maxFetchSize().

                                                                            {
   std::set<ChunkKey> optional_keys;
   auto total_chunk_size = getRequiredBuffersSize(chunk_key);
   auto max_size = maxFetchSize(chunk_key[CHUNK_KEY_DB_IDX]);
   // Add keys to the list of optional keys starting with the same fragment.  If we run out
   // of space, then exit early with what we have added so far.
   for (const auto& keys : {same_fragment_keys, diff_fragment_keys}) {
     for (const auto& key : keys) {
       auto column_keys = get_column_key_set(key);
       for (const auto& column_key : column_keys) {
         total_chunk_size += getBufferSize(column_key);
       }
       // Early exist if we exceed the size limit.
       if (total_chunk_size > max_size) {
         return optional_keys;
       }
       for (const auto& column_key : column_keys) {
         optional_keys.emplace(column_key);
       }
     }
   }
   return optional_keys;
 }

Here is the call graph for this function:

size_t foreign_storage::CachingForeignStorageMgr::getRequiredBuffersSize ( const ChunkKey & chunk_key ) const

private

Definition at line 407 of file CachingForeignStorageMgr.cpp.

References foreign_storage::get_column_key_set(), and getBufferSize().

Referenced by fetchBuffer(), and getOptionalKeysWithinSizeLimit().

                                                                                        {
   auto key_set = get_column_key_set(chunk_key);
   size_t total_size = 0U;
   for (const auto& key : key_set) {
     total_size += getBufferSize(key);
   }
   return total_size;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::CachingForeignStorageMgr::hasMaxFetchSize ( ) const

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 403 of file CachingForeignStorageMgr.cpp.

                                                      {
   return true;
 }

bool foreign_storage::CachingForeignStorageMgr::hasStoredDataWrapper	(	int32_t	db,
		int32_t	tb
	)		const

Definition at line 216 of file CachingForeignStorageMgr.cpp.

References disk_cache_, and foreign_storage::ForeignStorageCache::hasStoredDataWrapperMetadata().

                                                                                 {
   return disk_cache_->hasStoredDataWrapperMetadata(db, tb);
 }

Here is the call graph for this function:

size_t foreign_storage::CachingForeignStorageMgr::maxFetchSize ( int32_t db_id ) const

overrideprivatevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 399 of file CachingForeignStorageMgr.cpp.

References disk_cache_, and foreign_storage::ForeignStorageCache::getMaxChunkDataSize().

Referenced by fetchBuffer(), and getOptionalKeysWithinSizeLimit().

                                                                  {
   return disk_cache_->getMaxChunkDataSize();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::populateChunkBuffersSafely	(	ForeignDataWrapper &	data_wrapper,
		ChunkToBufferMap &	required_buffers,
		ChunkToBufferMap &	optional_buffers
	)

private

Definition at line 42 of file CachingForeignStorageMgr.cpp.

References CHECK_GT, foreign_storage::ForeignStorageCache::checkpoint(), disk_cache_, get_table_prefix(), foreign_storage::ForeignDataWrapper::populateChunkBuffers(), and foreign_storage::ForeignStorageMgr::updateFragmenterMetadata().

Referenced by fetchBuffer(), and refreshChunksInCacheByFragment().

                                         {
   CHECK_GT(required_buffers.size(), 0U) << "Must populate at least one buffer";
   try {
     ChunkSizeValidator chunk_size_validator(required_buffers.begin()->first);
     data_wrapper.populateChunkBuffers(required_buffers, optional_buffers);
     chunk_size_validator.validateChunkSizes(required_buffers);
     chunk_size_validator.validateChunkSizes(optional_buffers);
     updateFragmenterMetadata(required_buffers);
     updateFragmenterMetadata(optional_buffers);
   } catch (const std::runtime_error& error) {
     // clear any partially loaded but failed chunks (there may be some
     // fully-loaded chunks as well but they will be cleared conservatively
     // anyways)
     for (const auto& [chunk_key, buffer] : required_buffers) {
       if (auto file_buffer = dynamic_cast<File_Namespace::FileBuffer*>(buffer)) {
         file_buffer->freeChunkPages();
       }
     }
     for (const auto& [chunk_key, buffer] : optional_buffers) {
       if (auto file_buffer = dynamic_cast<File_Namespace::FileBuffer*>(buffer)) {
         file_buffer->freeChunkPages();
       }
     }
 
     throw ForeignStorageException(error.what());
   }
   // All required buffers should be from the same table.
   auto [db, tb] = get_table_prefix(required_buffers.begin()->first);
   disk_cache_->checkpoint(db, tb);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache	(	const ChunkKey &	table_key,
		const std::vector< ChunkKey > &	old_chunk_keys
	)

private

Definition at line 272 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, disk_cache_, getChunkMetadataVecFromDataWrapper(), getHighestCachedFragId(), is_table_key(), and refreshChunksInCacheByFragment().

Referenced by refreshTableInCache().

                                                {
   CHECK(is_table_key(table_key));
   int last_frag_id = getHighestCachedFragId(table_key);
 
   ChunkMetadataVector storage_metadata;
   getChunkMetadataVecFromDataWrapper(storage_metadata, table_key);
   try {
     disk_cache_->cacheMetadataVec(storage_metadata);
     refreshChunksInCacheByFragment(old_chunk_keys, last_frag_id);
   } catch (std::runtime_error& e) {
     throw PostEvictionRefreshException(e);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment	(	const std::vector< ChunkKey > &	old_chunk_keys,
		int	last_frag_id
	)

private

Definition at line 304 of file CachingForeignStorageMgr.cpp.

References CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, disk_cache_, get_table_key(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), foreign_storage::ForeignStorageCache::getChunkBuffersForCaching(), foreign_storage::ForeignStorageMgr::getDataWrapper(), is_varlen_data_key(), is_varlen_key(), foreign_storage::ForeignStorageCache::isMetadataCached(), LOG, foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::MAX_REFRESH_TIME_IN_SECONDS, populateChunkBuffersSafely(), and logger::WARNING.

Referenced by refreshAppendTableInCache(), and refreshNonAppendTableInCache().

                        {
   int64_t total_time{0};
   auto fragment_refresh_start_time = std::chrono::high_resolution_clock::now();
 
   if (old_chunk_keys.empty()) {
     return;
   }
   // Iterate through previously cached chunks and re-cache them. Caching is
   // done one fragment at a time, for all applicable chunks in the fragment.
   ChunkToBufferMap optional_buffers;
   std::set<ChunkKey> chunk_keys_to_be_cached;
   auto fragment_id = old_chunk_keys[0][CHUNK_KEY_FRAGMENT_IDX];
   const ChunkKey table_key{get_table_key(old_chunk_keys[0])};
   std::set<ChunkKey> chunk_keys_in_fragment;
   for (const auto& chunk_key : old_chunk_keys) {
     CHECK(chunk_key[CHUNK_KEY_TABLE_IDX] == table_key[CHUNK_KEY_TABLE_IDX]);
     if (chunk_key[CHUNK_KEY_FRAGMENT_IDX] < start_frag_id) {
       continue;
     }
     if (disk_cache_->isMetadataCached(chunk_key)) {
       if (chunk_key[CHUNK_KEY_FRAGMENT_IDX] != fragment_id) {
         if (chunk_keys_in_fragment.size() > 0) {
           auto required_buffers =
               disk_cache_->getChunkBuffersForCaching(chunk_keys_in_fragment);
           populateChunkBuffersSafely(
               *getDataWrapper(table_key), required_buffers, optional_buffers);
           chunk_keys_in_fragment.clear();
         }
         // At this point, cache buffers for refreshable chunks in the last fragment
         // have been populated. Exit if the max refresh time has been exceeded.
         // Otherwise, move to the next fragment.
         auto current_time = std::chrono::high_resolution_clock::now();
         total_time += std::chrono::duration_cast<std::chrono::seconds>(
                           current_time - fragment_refresh_start_time)
                           .count();
         if (total_time >= MAX_REFRESH_TIME_IN_SECONDS) {
           LOG(WARNING) << "Refresh time exceeded for table key: { " << table_key[0]
                        << ", " << table_key[1] << " } after fragment id: " << fragment_id;
           break;
         } else {
           fragment_refresh_start_time = std::chrono::high_resolution_clock::now();
         }
         fragment_id = chunk_key[CHUNK_KEY_FRAGMENT_IDX];
       }
       // Key may have been cached during scan
       if (disk_cache_->getCachedChunkIfExists(chunk_key) == nullptr) {
         if (is_varlen_key(chunk_key)) {
           CHECK(is_varlen_data_key(chunk_key));
           ChunkKey index_chunk_key{chunk_key[CHUNK_KEY_DB_IDX],
                                    chunk_key[CHUNK_KEY_TABLE_IDX],
                                    chunk_key[CHUNK_KEY_COLUMN_IDX],
                                    chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                                    2};
           chunk_keys_in_fragment.emplace(index_chunk_key);
           chunk_keys_to_be_cached.emplace(index_chunk_key);
         }
         chunk_keys_in_fragment.emplace(chunk_key);
         chunk_keys_to_be_cached.emplace(chunk_key);
       }
     }
   }
   if (chunk_keys_in_fragment.size() > 0) {
     auto required_buffers =
         disk_cache_->getChunkBuffersForCaching(chunk_keys_in_fragment);
     populateChunkBuffersSafely(
         *getDataWrapper(table_key), required_buffers, optional_buffers);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache	(	const ChunkKey &	table_key,
		const std::vector< ChunkKey > &	old_chunk_keys
	)

private

Definition at line 288 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, clearTable(), disk_cache_, getChunkMetadataVecFromDataWrapper(), is_table_key(), and refreshChunksInCacheByFragment().

Referenced by refreshTableInCache().

                                                {
   CHECK(is_table_key(table_key));
   ChunkMetadataVector storage_metadata;
   clearTable(table_key);
   getChunkMetadataVecFromDataWrapper(storage_metadata, table_key);
 
   try {
     disk_cache_->cacheMetadataVec(storage_metadata);
     refreshChunksInCacheByFragment(old_chunk_keys, 0);
   } catch (std::runtime_error& e) {
     throw PostEvictionRefreshException(e);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshTable	(	const ChunkKey &	table_key,
		const bool	evict_cached_entries
	)

overridevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 204 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageMgr::checkIfS3NeedsToBeEnabled(), clearTable(), foreign_storage::ForeignStorageMgr::clearTempChunkBufferMapEntriesForTable(), is_table_key(), and refreshTableInCache().

                                                                              {
   CHECK(is_table_key(table_key));
   ForeignStorageMgr::checkIfS3NeedsToBeEnabled(table_key);
   clearTempChunkBufferMapEntriesForTable(table_key);
   if (evict_cached_entries) {
     clearTable(table_key);
   } else {
     refreshTableInCache(table_key);
   }
 }

Here is the call graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshTableInCache ( const ChunkKey & table_key )

private

Definition at line 220 of file CachingForeignStorageMgr.cpp.

References CHECK, disk_cache_, foreign_storage::fragment_maps_to_leaf(), foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix(), foreign_storage::is_append_table_chunk_key(), foreign_storage::is_shardable_key(), is_table_key(), refreshAppendTableInCache(), refreshNonAppendTableInCache(), and show_chunk().

Referenced by refreshTable().

                                                                             {
   CHECK(is_table_key(table_key));
 
   // Preserve the list of which chunks were cached per table to refresh after clear.
   std::vector<ChunkKey> old_chunk_keys =
       disk_cache_->getCachedChunksForKeyPrefix(table_key);
 
   // Assert all data in cache is mapped to this leaf node in distributed.
   if (is_shardable_key(table_key)) {
     for (auto& key : old_chunk_keys) {
       CHECK(fragment_maps_to_leaf(key)) << show_chunk(key);
     }
   }
 
   auto append_mode = is_append_table_chunk_key(table_key);
 
   append_mode ? refreshAppendTableInCache(table_key, old_chunk_keys)
               : refreshNonAppendTableInCache(table_key, old_chunk_keys);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::removeTableRelatedDS	(	const int	db_id,
		const int	table_id
	)

override

Definition at line 394 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::clearForTablePrefix(), disk_cache_, and foreign_storage::ForeignStorageMgr::removeTableRelatedDS().

                                                                                        {
   disk_cache_->clearForTablePrefix({db_id, table_id});
   ForeignStorageMgr::removeTableRelatedDS(db_id, table_id);
 }

Here is the call graph for this function:

Member Data Documentation

ForeignStorageCache* foreign_storage::CachingForeignStorageMgr::disk_cache_

private

Definition at line 84 of file CachingForeignStorageMgr.h.

Referenced by CachingForeignStorageMgr(), clearTable(), createDataWrapperIfNotExists(), eraseDataWrapper(), fetchBuffer(), getBufferSize(), getChunkMetadataVecForKeyPrefix(), getChunkMetadataVecFromDataWrapper(), getHighestCachedFragId(), hasStoredDataWrapper(), maxFetchSize(), populateChunkBuffersSafely(), refreshAppendTableInCache(), refreshChunksInCacheByFragment(), refreshNonAppendTableInCache(), refreshTableInCache(), and removeTableRelatedDS().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/CachingForeignStorageMgr.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/CachingForeignStorageMgr.cpp

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation