foreign_storage::ForeignStorageCache Class Reference

#include <ForeignStorageCache.h>

Public Member Functions
	ForeignStorageCache (const DiskCacheConfig &config)
void	cacheTableChunks (const std::vector< ChunkKey > &chunk_keys)
void	cacheChunk (const ChunkKey &, AbstractBuffer *)
AbstractBuffer *	getCachedChunkIfExists (const ChunkKey &)
bool	isMetadataCached (const ChunkKey &) const
void	cacheMetadataVec (const ChunkMetadataVector &)
void	getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const
bool	hasCachedMetadataForKeyPrefix (const ChunkKey &) const
void	clearForTablePrefix (const ChunkKey &)
void	clear ()
std::vector< ChunkKey >	getCachedChunksForKeyPrefix (const ChunkKey &) const
bool	recoverCacheForTable (ChunkMetadataVector &, const ChunkKey &)
std::map< ChunkKey, AbstractBuffer * >	getChunkBuffersForCaching (const std::vector< ChunkKey > &chunk_keys) const
AbstractBuffer *	getChunkBufferForPrecaching (const ChunkKey &chunk_key, bool is_new_buffer)
void	deleteBufferIfExists (const ChunkKey &chunk_key)
size_t	getNumCachedChunks () const
size_t	getNumCachedMetadata () const
size_t	getNumChunksAdded () const
size_t	getNumMetadataAdded () const
std::string	dumpCachedChunkEntries () const
std::string	dumpCachedMetadataEntries () const
File_Namespace::GlobalFileMgr *	getGlobalFileMgr () const
std::string	getCacheDirectoryForTablePrefix (const ChunkKey &) const
void	cacheMetadataWithFragIdGreaterOrEqualTo (const ChunkMetadataVector &metadata_vec, const int frag_id)
void	evictThenEraseChunk (const ChunkKey &)

Private Member Functions
std::set< ChunkKey >::iterator	eraseChunk (const std::set< ChunkKey >::iterator &)
void	eraseChunk (const ChunkKey &chunk_key)
std::set< ChunkKey >::iterator	evictChunkByIterator (const std::set< ChunkKey >::iterator &chunk_it)
void	evictThenEraseChunkUnlocked (const ChunkKey &)
void	validatePath (const std::string &) const

Private Attributes
std::unique_ptr < File_Namespace::GlobalFileMgr >	global_file_mgr_
std::set< ChunkKey >	cached_chunks_
std::set< ChunkKey >	cached_metadata_
size_t	num_chunks_added_
size_t	num_metadata_added_
mapd_shared_mutex	chunks_mutex_
mapd_shared_mutex	metadata_mutex_

Detailed Description

Definition at line 58 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache

(

const DiskCacheConfig &

config

)

Definition at line 54 of file ForeignStorageCache.cpp.

References global_file_mgr_, DiskCacheConfig::num_reader_threads, DiskCacheConfig::path, and validatePath().

    : num_chunks_added_(0), num_metadata_added_(0) {
  validatePath(config.path);
  global_file_mgr_ = std::make_unique<File_Namespace::GlobalFileMgr>(
      0, config.path, config.num_reader_threads);
}

Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheChunk	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	buffer
	)

Definition at line 71 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, CHECK, chunks_mutex_, DEBUG_TIMER, global_file_mgr_, Data_Namespace::AbstractBuffer::isDirty(), metadata_mutex_, num_chunks_added_, and Data_Namespace::AbstractBuffer::setUpdated().

                                                                                      {
  auto timer = DEBUG_TIMER(__func__);
  write_lock meta_lock(metadata_mutex_);
  write_lock chunk_lock(chunks_mutex_);
  // We should only be caching buffers that are in sync with storage.
  CHECK(!buffer->isDirty());
  buffer->setUpdated();
  num_chunks_added_++;
  global_file_mgr_->putBuffer(chunk_key, buffer);
  global_file_mgr_->checkpoint();
  cached_metadata_.emplace(chunk_key);
  cached_chunks_.emplace(chunk_key);
  CHECK(!buffer->isDirty());
}

Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataVec

(

const ChunkMetadataVector &

metadata_vec

)

Definition at line 168 of file ForeignStorageCache.cpp.

References cached_metadata_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, evictThenEraseChunkUnlocked(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), global_file_mgr_, is_varlen_data_key(), is_varlen_key(), metadata_mutex_, num_metadata_added_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by cacheMetadataWithFragIdGreaterOrEqualTo(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

                                                                                  {
  auto timer = DEBUG_TIMER(__func__);
  write_lock meta_lock(metadata_mutex_);
  write_lock chunk_lock(chunks_mutex_);
  for (auto& [chunk_key, metadata] : metadata_vec) {
    cached_metadata_.emplace(chunk_key);
    AbstractBuffer* buf;
    AbstractBuffer* index_buffer = nullptr;
    ChunkKey index_chunk_key;
    if (is_varlen_key(chunk_key)) {
      // For variable length chunks, metadata is associated with the data chunk.
      CHECK(is_varlen_data_key(chunk_key));
      index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
                         chunk_key[CHUNK_KEY_TABLE_IDX],
                         chunk_key[CHUNK_KEY_COLUMN_IDX],
                         chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                         2};
    }
    bool chunk_in_cache = false;
    if (!global_file_mgr_->isBufferOnDevice(chunk_key)) {
      buf = global_file_mgr_->createBuffer(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(!global_file_mgr_->isBufferOnDevice(index_chunk_key));
        index_buffer = global_file_mgr_->createBuffer(index_chunk_key);
        CHECK(index_buffer);
      }
    } else {
      buf = global_file_mgr_->getBuffer(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(global_file_mgr_->isBufferOnDevice(index_chunk_key));
        index_buffer = global_file_mgr_->getBuffer(index_chunk_key);
        CHECK(index_buffer);
      }

      // We should have already cleared the data unless we are appending
      // If the buffer metadata has changed, we need to remove this chunk
      if (buf->getEncoder() != nullptr) {
        const std::shared_ptr<ChunkMetadata> buf_metadata =
            std::make_shared<ChunkMetadata>();
        buf->getEncoder()->getMetadata(buf_metadata);
        chunk_in_cache = *metadata.get() == *buf_metadata;
      }
    }

    if (!chunk_in_cache) {
      set_metadata_for_buffer(buf, metadata.get());
      evictThenEraseChunkUnlocked(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(index_buffer);
        index_buffer->setUpdated();
        evictThenEraseChunkUnlocked(index_chunk_key);
      }
    }
    num_metadata_added_++;
  }
  global_file_mgr_->checkpoint();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataWithFragIdGreaterOrEqualTo	(	const ChunkMetadataVector &	metadata_vec,
		const int	frag_id
	)

Definition at line 401 of file ForeignStorageCache.cpp.

References cacheMetadataVec(), and CHUNK_KEY_FRAGMENT_IDX.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache().

                       {
  // Only re-cache last fragment and above
  ChunkMetadataVector new_metadata_vec;
  for (const auto& chunk_metadata : metadata_vec) {
    if (chunk_metadata.first[CHUNK_KEY_FRAGMENT_IDX] >= frag_id) {
      new_metadata_vec.push_back(chunk_metadata);
    }
  }
  cacheMetadataVec(new_metadata_vec);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheTableChunks

(

const std::vector< ChunkKey > &

chunk_keys

)

Caches the chunks for the given chunk keys. Chunk buffers for chunks to be cached are expected to have already been populated before calling this method. This method also expects all provided chunk keys to be for the same table.

Parameters

chunk_keys

- keys of chunks to be cached

Definition at line 86 of file ForeignStorageCache.cpp.

References cached_chunks_, CHECK, CHECK_EQ, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, global_file_mgr_, and num_chunks_added_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                                {
  auto timer = DEBUG_TIMER(__func__);
  write_lock lock(chunks_mutex_);
  CHECK(!chunk_keys.empty());

  auto db_id = chunk_keys[0][CHUNK_KEY_DB_IDX];
  auto table_id = chunk_keys[0][CHUNK_KEY_TABLE_IDX];
  const ChunkKey table_key{db_id, table_id};

  for (const auto& chunk_key : chunk_keys) {
    CHECK_EQ(db_id, chunk_key[CHUNK_KEY_DB_IDX]);
    CHECK_EQ(table_id, chunk_key[CHUNK_KEY_TABLE_IDX]);
    CHECK(global_file_mgr_->isBufferOnDevice(chunk_key));
    num_chunks_added_++;
    cached_chunks_.emplace(chunk_key);
  }
  global_file_mgr_->checkpoint(db_id, table_id);
}

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear

(

)

Definition at line 284 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), global_file_mgr_, and metadata_mutex_.

                                {
  auto timer = DEBUG_TIMER(__func__);
  std::set<ChunkKey> table_keys;
  {
    write_lock w_lock(chunks_mutex_);
    for (auto chunk_it = cached_chunks_.begin(); chunk_it != cached_chunks_.end();) {
      chunk_it = evictChunkByIterator(chunk_it);
    }
  }
  {
    write_lock w_lock(metadata_mutex_);
    for (auto meta_it = cached_metadata_.begin(); meta_it != cached_metadata_.end();) {
      table_keys.emplace(ChunkKey{(*meta_it)[0], (*meta_it)[1]});
      meta_it = cached_metadata_.erase(meta_it);
    }
  }
  for (const auto& table_key : table_keys) {
    global_file_mgr_->removeTableRelatedDS(table_key[0], table_key[1]);
  }
}

Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::clearForTablePrefix

(

const ChunkKey &

chunk_prefix

)

Definition at line 260 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, CHECK, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), global_file_mgr_, is_table_key(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache(), and foreign_storage::CachingForeignStorageMgr::refreshTable().

                                                                          {
  CHECK(is_table_key(chunk_prefix));
  auto timer = DEBUG_TIMER(__func__);
  ChunkKey upper_prefix(chunk_prefix);
  upper_prefix.push_back(std::numeric_limits<int>::max());
  {
    write_lock w_lock(chunks_mutex_);
    // Delete chunks for prefix
    auto end_it = cached_chunks_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
    for (auto chunk_it = cached_chunks_.lower_bound(chunk_prefix); chunk_it != end_it;) {
      chunk_it = evictChunkByIterator(chunk_it);
    }
  }
  {
    write_lock w_lock(metadata_mutex_);
    // Delete metadata for prefix
    auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
    for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;) {
      meta_it = cached_metadata_.erase(meta_it);
    }
  }
  global_file_mgr_->removeTableRelatedDS(chunk_prefix[0], chunk_prefix[1]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists

(

const ChunkKey &

chunk_key

)

Definition at line 61 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, chunks_mutex_, global_file_mgr_, and metadata_mutex_.

                                                                        {
  write_lock meta_lock(metadata_mutex_);
  write_lock chunk_lock(chunks_mutex_);
  if (cached_metadata_.find(chunk_key) != cached_metadata_.end()) {
    global_file_mgr_->deleteBuffer(chunk_key);
    cached_chunks_.erase(chunk_key);
    cached_metadata_.erase(chunk_key);
  }
}

std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries

(

)

const

Definition at line 354 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, and show_chunk().

                                                            {
  auto timer = DEBUG_TIMER(__func__);
  std::string ret_string = "Cached chunks:\n";
  for (const auto& chunk_key : cached_chunks_) {
    ret_string += "  " + show_chunk(chunk_key) + "\n";
  }
  return ret_string;
}

Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries

(

)

const

Definition at line 363 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and show_chunk().

                                                               {
  auto timer = DEBUG_TIMER(__func__);
  std::string ret_string = "Cached ChunkMetadata:\n";
  for (const auto& meta_key : cached_metadata_) {
    ret_string += "  " + show_chunk(meta_key) + "\n";
  }
  return ret_string;
}

Here is the call graph for this function:

std::set<ChunkKey>::iterator foreign_storage::ForeignStorageCache::eraseChunk ( const std::set< ChunkKey >::iterator &  )

private

Referenced by evictThenEraseChunkUnlocked().

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey &  chunk_key )

private

Definition at line 334 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, File_Namespace::FileBuffer::freeChunkPages(), and global_file_mgr_.

                                                              {
  auto timer = DEBUG_TIMER(__func__);
  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
    return;
  }
  File_Namespace::FileBuffer* file_buffer =
      static_cast<File_Namespace::FileBuffer*>(global_file_mgr_->getBuffer(chunk_key));
  file_buffer->freeChunkPages();
  cached_chunks_.erase(chunk_key);
}

Here is the call graph for this function:

std::set< ChunkKey >::iterator foreign_storage::ForeignStorageCache::evictChunkByIterator ( const std::set< ChunkKey >::iterator &  chunk_it )

private

Definition at line 345 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, File_Namespace::FileBuffer::freeChunkPages(), and global_file_mgr_.

Referenced by clear(), and clearForTablePrefix().

                                              {
  auto timer = DEBUG_TIMER(__func__);
  File_Namespace::FileBuffer* file_buffer =
      static_cast<File_Namespace::FileBuffer*>(global_file_mgr_->getBuffer(*chunk_it));
  file_buffer->freeChunkPages();
  return cached_chunks_.erase(chunk_it);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunk

(

const ChunkKey &

chunk_key

)

Definition at line 158 of file ForeignStorageCache.cpp.

References chunks_mutex_, and evictThenEraseChunkUnlocked().

                                                                       {
  write_lock chunk_lock(chunks_mutex_);
  evictThenEraseChunkUnlocked(chunk_key);
}

Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunkUnlocked ( const ChunkKey &  chunk_key )

private

Definition at line 163 of file ForeignStorageCache.cpp.

References eraseChunk(), and get_table_key().

Referenced by cacheMetadataVec(), and evictThenEraseChunk().

                                                                               {
  const ChunkKey table_prefix = get_table_key(chunk_key);
  eraseChunk(chunk_key);
}

Here is the call graph for this function:

Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists

(

const ChunkKey &

chunk_key

)

Definition at line 105 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, DEBUG_TIMER, and global_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                                     {
  auto timer = DEBUG_TIMER(__func__);
  {
    read_lock lock(chunks_mutex_);
    // We do this instead of calling getBuffer so that we don't create a fileMgr if the
    // chunk doesn't exist.
    if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
      return nullptr;
    }
  }
  return global_file_mgr_->getBuffer(chunk_key);
}

Here is the caller graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix

(

const ChunkKey &

chunk_prefix

)

const

Definition at line 305 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                        {
  read_lock r_lock(chunks_mutex_);
  std::vector<ChunkKey> ret_vec;
  iterate_over_matching_prefix(
      [&ret_vec](auto chunk) { ret_vec.push_back(chunk); }, cached_chunks_, chunk_prefix);
  return ret_vec;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTablePrefix

(

const ChunkKey &

table_prefix

)

const

Definition at line 392 of file ForeignStorageCache.cpp.

References CHECK, File_Namespace::GlobalFileMgr::getFileMgr(), and getGlobalFileMgr().

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk().

                                        {
  CHECK(table_prefix.size() >= 2);
  auto fileMgr = dynamic_cast<File_Namespace::FileMgr*>(
      getGlobalFileMgr()->getFileMgr(table_prefix));
  CHECK(fileMgr);
  return fileMgr->getFileMgrBasePath();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix	(	ChunkMetadataVector &	metadata_vec,
		const ChunkKey &	chunk_prefix
	)		const

Definition at line 229 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, global_file_mgr_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

                                        {
  auto timer = DEBUG_TIMER(__func__);
  read_lock r_lock(metadata_mutex_);
  iterate_over_matching_prefix(
      [&metadata_vec, this](auto chunk) {
        std::shared_ptr<ChunkMetadata> buf_metadata = std::make_shared<ChunkMetadata>();
        global_file_mgr_->getBuffer(chunk)->getEncoder()->getMetadata(buf_metadata);
        metadata_vec.push_back(std::make_pair(chunk, buf_metadata));
      },
      cached_metadata_,
      chunk_prefix);
}

Here is the call graph for this function:

Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getChunkBufferForPrecaching	(	const ChunkKey &	chunk_key,
		bool	is_new_buffer
	)

Definition at line 414 of file ForeignStorageCache.cpp.

References CHECK, File_Namespace::GlobalFileMgr::createBuffer(), File_Namespace::GlobalFileMgr::getBuffer(), and getGlobalFileMgr().

                        {
  if (!is_new_buffer) {
    CHECK(getGlobalFileMgr()->isBufferOnDevice(chunk_key));
    return getGlobalFileMgr()->getBuffer(chunk_key);
  } else {
    CHECK(!getGlobalFileMgr()->isBufferOnDevice(chunk_key));
    return getGlobalFileMgr()->createBuffer(chunk_key);
  }
}

Here is the call graph for this function:

std::map< ChunkKey, AbstractBuffer * > foreign_storage::ForeignStorageCache::getChunkBuffersForCaching

(

const std::vector< ChunkKey > &

chunk_keys

)

const

Definition at line 314 of file ForeignStorageCache.cpp.

References cached_chunks_, CHECK, CHECK_EQ, chunks_mutex_, DEBUG_TIMER, and global_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                 {
  auto timer = DEBUG_TIMER(__func__);
  std::map<ChunkKey, AbstractBuffer*> chunk_buffer_map;
  read_lock lock(chunks_mutex_);
  for (const auto& chunk_key : chunk_keys) {
    CHECK(cached_chunks_.find(chunk_key) == cached_chunks_.end());
    CHECK(global_file_mgr_->isBufferOnDevice(chunk_key));
    chunk_buffer_map[chunk_key] = global_file_mgr_->getBuffer(chunk_key);
    CHECK(dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[chunk_key]));
    CHECK_EQ(chunk_buffer_map[chunk_key]->pageCount(), static_cast<size_t>(0));

    // Clear all buffer metadata
    chunk_buffer_map[chunk_key]->resetToEmpty();
  }
  return chunk_buffer_map;
}

Here is the caller graph for this function:

File_Namespace::GlobalFileMgr* foreign_storage::ForeignStorageCache::getGlobalFileMgr ( ) const

inline

Definition at line 100 of file ForeignStorageCache.h.

Referenced by getCacheDirectoryForTablePrefix(), and getChunkBufferForPrecaching().

                                                               {
    return global_file_mgr_.get();
  }

Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const

inline

Definition at line 91 of file ForeignStorageCache.h.

{ return cached_chunks_.size(); }

size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const

inline

Definition at line 92 of file ForeignStorageCache.h.

{ return cached_metadata_.size(); }

size_t foreign_storage::ForeignStorageCache::getNumChunksAdded ( ) const

inline

Definition at line 93 of file ForeignStorageCache.h.

{ return num_chunks_added_; }

size_t foreign_storage::ForeignStorageCache::getNumMetadataAdded ( ) const

inline

Definition at line 94 of file ForeignStorageCache.h.

{ return num_metadata_added_; }

bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix

(

const ChunkKey &

chunk_prefix

)

const

Definition at line 244 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                        {
  auto timer = DEBUG_TIMER(__func__);
  read_lock lock(metadata_mutex_);
  // We don't use iterateOvermatchingPrefix() here because we want to exit early if
  // possible.
  ChunkKey upper_prefix(chunk_prefix);
  upper_prefix.push_back(std::numeric_limits<int>::max());
  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;
       ++meta_it) {
    return true;
  }
  return false;
}

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached

(

const ChunkKey &

chunk_key

)

const

Definition at line 118 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                          {
  auto timer = DEBUG_TIMER(__func__);
  read_lock lock(metadata_mutex_);
  return (cached_metadata_.find(chunk_key) != cached_metadata_.end());
}

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::recoverCacheForTable	(	ChunkMetadataVector &	meta_vec,
		const ChunkKey &	table_key
	)

Definition at line 124 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, global_file_mgr_, is_table_key(), is_varlen_data_key(), and is_varlen_key().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                                          {
  write_lock lock(chunks_mutex_);
  CHECK(meta_vec.size() == 0);
  CHECK(is_table_key(table_key));
  CHECK(dynamic_cast<File_Namespace::FileMgr*>(global_file_mgr_->getFileMgr(table_key)));

  global_file_mgr_->getChunkMetadataVecForKeyPrefix(meta_vec, table_key);
  for (auto& [chunk_key, metadata] : meta_vec) {
    cached_metadata_.emplace(chunk_key);
    // If there is no page count then the chunk was metadata only and should not be
    // cached.
    if (const auto& buf = global_file_mgr_->getBuffer(chunk_key); buf->pageCount() > 0) {
      cached_chunks_.emplace(chunk_key);
    }

    if (is_varlen_key(chunk_key)) {
      // Metadata is only available for the data chunk, but look for the index as well
      CHECK(is_varlen_data_key(chunk_key));
      ChunkKey index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
                                  chunk_key[CHUNK_KEY_TABLE_IDX],
                                  chunk_key[CHUNK_KEY_COLUMN_IDX],
                                  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                                  2};

      if (const auto& buf = global_file_mgr_->getBuffer(index_chunk_key);
          buf->pageCount() > 0) {
        cached_chunks_.emplace(index_chunk_key);
      }
    }
  }
  return (meta_vec.size() > 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string &  base_path ) const

private

Definition at line 372 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

                                                                       {
  // check if base_path already exists, and if not create one
  boost::filesystem::path path(base_path);
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      throw std::runtime_error{
          "cache path \"" + base_path +
          "\" is not a directory.  Please specify a valid directory "
          "with --disk_cache_path=<path>, or use the default location."};
    }
  } else {  // data directory does not exist
    if (!boost::filesystem::create_directory(path)) {
      throw std::runtime_error{
          "could not create directory at cache path \"" + base_path +
          "\".  Please specify a valid directory location "
          "with --disk_cache_path=<path> or use the default location."};
    }
  }
}

Here is the caller graph for this function:

Member Data Documentation

std::set<ChunkKey> foreign_storage::ForeignStorageCache::cached_chunks_

private

Definition at line 122 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheTableChunks(), clear(), clearForTablePrefix(), deleteBufferIfExists(), dumpCachedChunkEntries(), eraseChunk(), evictChunkByIterator(), getCachedChunkIfExists(), getCachedChunksForKeyPrefix(), getChunkBuffersForCaching(), and recoverCacheForTable().

std::set<ChunkKey> foreign_storage::ForeignStorageCache::cached_metadata_

private

Definition at line 123 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), clear(), clearForTablePrefix(), deleteBufferIfExists(), dumpCachedMetadataEntries(), getCachedMetadataVecForKeyPrefix(), hasCachedMetadataForKeyPrefix(), isMetadataCached(), and recoverCacheForTable().

mapd_shared_mutex foreign_storage::ForeignStorageCache::chunks_mutex_

mutableprivate

Definition at line 130 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), cacheTableChunks(), clear(), clearForTablePrefix(), deleteBufferIfExists(), evictThenEraseChunk(), getCachedChunkIfExists(), getCachedChunksForKeyPrefix(), getChunkBuffersForCaching(), and recoverCacheForTable().

std::unique_ptr<File_Namespace::GlobalFileMgr> foreign_storage::ForeignStorageCache::global_file_mgr_

private

Definition at line 119 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), cacheTableChunks(), clear(), clearForTablePrefix(), deleteBufferIfExists(), eraseChunk(), evictChunkByIterator(), ForeignStorageCache(), getCachedChunkIfExists(), getCachedMetadataVecForKeyPrefix(), getChunkBuffersForCaching(), and recoverCacheForTable().

mapd_shared_mutex foreign_storage::ForeignStorageCache::metadata_mutex_

mutableprivate

Definition at line 131 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), clear(), clearForTablePrefix(), deleteBufferIfExists(), getCachedMetadataVecForKeyPrefix(), hasCachedMetadataForKeyPrefix(), and isMetadataCached().

size_t foreign_storage::ForeignStorageCache::num_chunks_added_

private

Definition at line 126 of file ForeignStorageCache.h.

Referenced by cacheChunk(), and cacheTableChunks().

size_t foreign_storage::ForeignStorageCache::num_metadata_added_

private

Definition at line 127 of file ForeignStorageCache.h.

Referenced by cacheMetadataVec().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.cpp