foreign_storage::ForeignStorageCache Class Reference

#include <ForeignStorageCache.h>

Public Member Functions
	ForeignStorageCache (const DiskCacheConfig &config)
void	cacheTableChunks (const std::vector< ChunkKey > &chunk_keys)
void	cacheChunk (const ChunkKey &, AbstractBuffer *)
AbstractBuffer *	getCachedChunkIfExists (const ChunkKey &)
bool	isMetadataCached (const ChunkKey &) const
void	cacheMetadataVec (const ChunkMetadataVector &)
void	getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const
bool	hasCachedMetadataForKeyPrefix (const ChunkKey &) const
void	clearForTablePrefix (const ChunkKey &)
void	clear ()
std::vector< ChunkKey >	getCachedChunksForKeyPrefix (const ChunkKey &) const
bool	recoverCacheForTable (ChunkMetadataVector &, const ChunkKey &)
ChunkToBufferMap	getChunkBuffersForCaching (const std::vector< ChunkKey > &chunk_keys) const
AbstractBuffer *	getChunkBufferForPrecaching (const ChunkKey &chunk_key, bool is_new_buffer)
void	deleteBufferIfExists (const ChunkKey &chunk_key)
size_t	getNumCachedChunks () const
size_t	getNumCachedMetadata () const
size_t	getNumChunksAdded () const
size_t	getNumMetadataAdded () const
std::string	dumpCachedChunkEntries () const
std::string	dumpCachedMetadataEntries () const
std::string	getCacheDirectory () const
std::string	getCacheDirectoryForTable (int db_id, int tb_id) const
void	cacheMetadataWithFragIdGreaterOrEqualTo (const ChunkMetadataVector &metadata_vec, const int frag_id)
void	evictThenEraseChunk (const ChunkKey &)
uint64_t	getSpaceReservedByTable (int db_id, int tb_id) const

Private Member Functions
std::set< ChunkKey >::iterator	eraseChunk (const std::set< ChunkKey >::iterator &)
void	eraseChunk (const ChunkKey &chunk_key)
std::set< ChunkKey >::iterator	evictChunkByIterator (const std::set< ChunkKey >::iterator &chunk_it)
void	evictThenEraseChunkUnlocked (const ChunkKey &)
void	validatePath (const std::string &) const

Private Attributes
std::unique_ptr < File_Namespace::CachingFileMgr >	caching_file_mgr_
std::set< ChunkKey >	cached_chunks_
std::set< ChunkKey >	cached_metadata_
size_t	num_chunks_added_
size_t	num_metadata_added_
mapd_shared_mutex	chunks_mutex_
mapd_shared_mutex	metadata_mutex_

Detailed Description

Definition at line 59 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache

(

const DiskCacheConfig &

config

)

Definition at line 54 of file ForeignStorageCache.cpp.

References caching_file_mgr_, DiskCacheConfig::num_reader_threads, DiskCacheConfig::path, and validatePath().

    : num_chunks_added_(0), num_metadata_added_(0) {
  validatePath(config.path);
  caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(
      config.path, config.num_reader_threads);
}

Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheChunk	(	const ChunkKey &	chunk_key,
		AbstractBuffer *	buffer
	)

Definition at line 71 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, chunks_mutex_, Data_Namespace::AbstractBuffer::isDirty(), metadata_mutex_, num_chunks_added_, and Data_Namespace::AbstractBuffer::setUpdated().

                                                                                      {
  write_lock meta_lock(metadata_mutex_);
  write_lock chunk_lock(chunks_mutex_);
  // We should only be caching buffers that are in sync with storage.
  CHECK(!buffer->isDirty());
  buffer->setUpdated();
  num_chunks_added_++;
  caching_file_mgr_->putBuffer(chunk_key, buffer);
  caching_file_mgr_->checkpoint();
  cached_metadata_.emplace(chunk_key);
  cached_chunks_.emplace(chunk_key);
  CHECK(!buffer->isDirty());
}

Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataVec

(

const ChunkMetadataVector &

metadata_vec

)

Definition at line 164 of file ForeignStorageCache.cpp.

References cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, evictThenEraseChunkUnlocked(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), is_varlen_data_key(), is_varlen_key(), metadata_mutex_, num_metadata_added_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by cacheMetadataWithFragIdGreaterOrEqualTo(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

                                                                                  {
  auto timer = DEBUG_TIMER(__func__);
  write_lock meta_lock(metadata_mutex_);
  write_lock chunk_lock(chunks_mutex_);
  for (auto& [chunk_key, metadata] : metadata_vec) {
    cached_metadata_.emplace(chunk_key);
    AbstractBuffer* buf;
    AbstractBuffer* index_buffer = nullptr;
    ChunkKey index_chunk_key;
    if (is_varlen_key(chunk_key)) {
      // For variable length chunks, metadata is associated with the data chunk.
      CHECK(is_varlen_data_key(chunk_key));
      index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
                         chunk_key[CHUNK_KEY_TABLE_IDX],
                         chunk_key[CHUNK_KEY_COLUMN_IDX],
                         chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                         2};
    }
    bool chunk_in_cache = false;
    if (!caching_file_mgr_->isBufferOnDevice(chunk_key)) {
      buf = caching_file_mgr_->createBuffer(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(!caching_file_mgr_->isBufferOnDevice(index_chunk_key));
        index_buffer = caching_file_mgr_->createBuffer(index_chunk_key);
        CHECK(index_buffer);
      }
    } else {
      buf = caching_file_mgr_->getBuffer(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(caching_file_mgr_->isBufferOnDevice(index_chunk_key));
        index_buffer = caching_file_mgr_->getBuffer(index_chunk_key);
        CHECK(index_buffer);
      }

      // We should have already cleared the data unless we are appending
      // If the buffer metadata has changed, we need to remove this chunk
      if (buf->getEncoder() != nullptr) {
        const std::shared_ptr<ChunkMetadata> buf_metadata =
            std::make_shared<ChunkMetadata>();
        buf->getEncoder()->getMetadata(buf_metadata);
        chunk_in_cache = *metadata.get() == *buf_metadata;
      }
    }

    if (!chunk_in_cache) {
      set_metadata_for_buffer(buf, metadata.get());
      evictThenEraseChunkUnlocked(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(index_buffer);
        index_buffer->setUpdated();
        evictThenEraseChunkUnlocked(index_chunk_key);
      }
    }
    num_metadata_added_++;
  }
  caching_file_mgr_->checkpoint();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataWithFragIdGreaterOrEqualTo	(	const ChunkMetadataVector &	metadata_vec,
		const int	frag_id
	)

Definition at line 385 of file ForeignStorageCache.cpp.

References cacheMetadataVec(), and CHUNK_KEY_FRAGMENT_IDX.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache().

                       {
  // Only re-cache last fragment and above
  ChunkMetadataVector new_metadata_vec;
  for (const auto& chunk_metadata : metadata_vec) {
    if (chunk_metadata.first[CHUNK_KEY_FRAGMENT_IDX] >= frag_id) {
      new_metadata_vec.push_back(chunk_metadata);
    }
  }
  cacheMetadataVec(new_metadata_vec);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheTableChunks

(

const std::vector< ChunkKey > &

chunk_keys

)

Caches the chunks for the given chunk keys. Chunk buffers for chunks to be cached are expected to have already been populated before calling this method. This method also expects all provided chunk keys to be for the same table.

Parameters

chunk_keys

- keys of chunks to be cached

Definition at line 85 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, CHECK, CHECK_EQ, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, and num_chunks_added_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                                {
  auto timer = DEBUG_TIMER(__func__);
  write_lock lock(chunks_mutex_);
  CHECK(!chunk_keys.empty());

  auto db_id = chunk_keys[0][CHUNK_KEY_DB_IDX];
  auto table_id = chunk_keys[0][CHUNK_KEY_TABLE_IDX];
  const ChunkKey table_key{db_id, table_id};

  for (const auto& chunk_key : chunk_keys) {
    CHECK_EQ(db_id, chunk_key[CHUNK_KEY_DB_IDX]);
    CHECK_EQ(table_id, chunk_key[CHUNK_KEY_TABLE_IDX]);
    CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
    num_chunks_added_++;
    cached_chunks_.emplace(chunk_key);
  }
  caching_file_mgr_->checkpoint();
}

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear

(

)

Definition at line 279 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), and metadata_mutex_.

                                {
  auto timer = DEBUG_TIMER(__func__);
  std::set<ChunkKey> table_keys;
  {
    write_lock w_lock(chunks_mutex_);
    for (auto chunk_it = cached_chunks_.begin(); chunk_it != cached_chunks_.end();) {
      chunk_it = evictChunkByIterator(chunk_it);
    }
  }
  {
    write_lock w_lock(metadata_mutex_);
    for (auto meta_it = cached_metadata_.begin(); meta_it != cached_metadata_.end();) {
      table_keys.emplace(ChunkKey{(*meta_it)[0], (*meta_it)[1]});
      meta_it = cached_metadata_.erase(meta_it);
    }
  }
  // FileMgrs do not clean up after themselves nicely, so we need to close all their disk
  // resources and then re-create the CachingFileMgr to reset it.
  caching_file_mgr_->closeRemovePhysical();
  boost::filesystem::create_directory(caching_file_mgr_->getFileMgrBasePath());
  caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(
      caching_file_mgr_->getFileMgrBasePath(), caching_file_mgr_->getNumReaderThreads());
}

Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::clearForTablePrefix

(

const ChunkKey &

chunk_prefix

)

Definition at line 255 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), is_table_key(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache(), and foreign_storage::CachingForeignStorageMgr::refreshTable().

                                                                          {
  CHECK(is_table_key(chunk_prefix));
  auto timer = DEBUG_TIMER(__func__);
  ChunkKey upper_prefix(chunk_prefix);
  upper_prefix.push_back(std::numeric_limits<int>::max());
  {
    write_lock w_lock(chunks_mutex_);
    // Delete chunks for prefix
    auto end_it = cached_chunks_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
    for (auto chunk_it = cached_chunks_.lower_bound(chunk_prefix); chunk_it != end_it;) {
      chunk_it = evictChunkByIterator(chunk_it);
    }
  }
  {
    write_lock w_lock(metadata_mutex_);
    // Delete metadata for prefix
    auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
    for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;) {
      meta_it = cached_metadata_.erase(meta_it);
    }
  }
  caching_file_mgr_->clearForTable(chunk_prefix[0], chunk_prefix[1]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists

(

const ChunkKey &

chunk_key

)

Definition at line 61 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, chunks_mutex_, and metadata_mutex_.

                                                                        {
  write_lock meta_lock(metadata_mutex_);
  write_lock chunk_lock(chunks_mutex_);
  if (cached_metadata_.find(chunk_key) != cached_metadata_.end()) {
    caching_file_mgr_->deleteBuffer(chunk_key);
    cached_chunks_.erase(chunk_key);
    cached_metadata_.erase(chunk_key);
  }
}

std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries

(

)

const

Definition at line 349 of file ForeignStorageCache.cpp.

References cached_chunks_, and show_chunk().

                                                            {
  std::string ret_string = "Cached chunks:\n";
  for (const auto& chunk_key : cached_chunks_) {
    ret_string += "  " + show_chunk(chunk_key) + "\n";
  }
  return ret_string;
}

Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries

(

)

const

Definition at line 357 of file ForeignStorageCache.cpp.

References cached_metadata_, and show_chunk().

                                                               {
  std::string ret_string = "Cached ChunkMetadata:\n";
  for (const auto& meta_key : cached_metadata_) {
    ret_string += "  " + show_chunk(meta_key) + "\n";
  }
  return ret_string;
}

Here is the call graph for this function:

std::set<ChunkKey>::iterator foreign_storage::ForeignStorageCache::eraseChunk ( const std::set< ChunkKey >::iterator &  )

private

Referenced by evictThenEraseChunkUnlocked().

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey &  chunk_key )

private

Definition at line 331 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and File_Namespace::FileBuffer::freeChunkPages().

                                                              {
  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
    return;
  }
  File_Namespace::FileBuffer* file_buffer =
      static_cast<File_Namespace::FileBuffer*>(caching_file_mgr_->getBuffer(chunk_key));
  file_buffer->freeChunkPages();
  cached_chunks_.erase(chunk_key);
}

Here is the call graph for this function:

std::set< ChunkKey >::iterator foreign_storage::ForeignStorageCache::evictChunkByIterator ( const std::set< ChunkKey >::iterator &  chunk_it )

private

Definition at line 341 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and File_Namespace::FileBuffer::freeChunkPages().

Referenced by clear(), and clearForTablePrefix().

                                              {
  File_Namespace::FileBuffer* file_buffer =
      static_cast<File_Namespace::FileBuffer*>(caching_file_mgr_->getBuffer(*chunk_it));
  file_buffer->freeChunkPages();
  return cached_chunks_.erase(chunk_it);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunk

(

const ChunkKey &

chunk_key

)

Definition at line 154 of file ForeignStorageCache.cpp.

References chunks_mutex_, and evictThenEraseChunkUnlocked().

                                                                       {
  write_lock chunk_lock(chunks_mutex_);
  evictThenEraseChunkUnlocked(chunk_key);
}

Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunkUnlocked ( const ChunkKey &  chunk_key )

private

Definition at line 159 of file ForeignStorageCache.cpp.

References eraseChunk(), and get_table_key().

Referenced by cacheMetadataVec(), and evictThenEraseChunk().

                                                                               {
  const ChunkKey table_prefix = get_table_key(chunk_key);
  eraseChunk(chunk_key);
}

Here is the call graph for this function:

Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists

(

const ChunkKey &

chunk_key

)

Definition at line 104 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and chunks_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                                     {
  {
    read_lock lock(chunks_mutex_);
    // We do this instead of calling getBuffer so that we don't create a fileMgr if the
    // chunk doesn't exist.
    if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
      return nullptr;
    }
  }
  return caching_file_mgr_->getBuffer(chunk_key);
}

Here is the caller graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix

(

const ChunkKey &

chunk_prefix

)

const

Definition at line 303 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                        {
  read_lock r_lock(chunks_mutex_);
  std::vector<ChunkKey> ret_vec;
  iterate_over_matching_prefix(
      [&ret_vec](auto chunk) { ret_vec.push_back(chunk); }, cached_chunks_, chunk_prefix);
  return ret_vec;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectory ( ) const

inline

Definition at line 101 of file ForeignStorageCache.h.

                                             {
    return caching_file_mgr_->getFileMgrBasePath();
  }

std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTable ( int  db_id, int  tb_id  ) const

inline

Definition at line 105 of file ForeignStorageCache.h.

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk().

                                                                         {
    return caching_file_mgr_->getOrAddTableDir(db_id, tb_id);
  }

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix	(	ChunkMetadataVector &	metadata_vec,
		const ChunkKey &	chunk_prefix
	)		const

Definition at line 225 of file ForeignStorageCache.cpp.

References cached_metadata_, caching_file_mgr_, DEBUG_TIMER, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

                                        {
  auto timer = DEBUG_TIMER(__func__);
  read_lock r_lock(metadata_mutex_);
  iterate_over_matching_prefix(
      [&metadata_vec, this](auto chunk) {
        std::shared_ptr<ChunkMetadata> buf_metadata = std::make_shared<ChunkMetadata>();
        caching_file_mgr_->getBuffer(chunk)->getEncoder()->getMetadata(buf_metadata);
        metadata_vec.push_back(std::make_pair(chunk, buf_metadata));
      },
      cached_metadata_,
      chunk_prefix);
}

Here is the call graph for this function:

Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getChunkBufferForPrecaching	(	const ChunkKey &	chunk_key,
		bool	is_new_buffer
	)

Definition at line 398 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and CHECK.

                        {
  if (!is_new_buffer) {
    CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
    return caching_file_mgr_->getBuffer(chunk_key);
  } else {
    CHECK(!caching_file_mgr_->isBufferOnDevice(chunk_key));
    return caching_file_mgr_->createBuffer(chunk_key);
  }
}

ChunkToBufferMap foreign_storage::ForeignStorageCache::getChunkBuffersForCaching

(

const std::vector< ChunkKey > &

chunk_keys

)

const

Definition at line 312 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, CHECK, CHECK_EQ, and chunks_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                 {
  ChunkToBufferMap chunk_buffer_map;
  read_lock lock(chunks_mutex_);
  for (const auto& chunk_key : chunk_keys) {
    CHECK(cached_chunks_.find(chunk_key) == cached_chunks_.end());
    CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
    chunk_buffer_map[chunk_key] = caching_file_mgr_->getBuffer(chunk_key);
    CHECK(dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[chunk_key]));
    CHECK_EQ(chunk_buffer_map[chunk_key]->pageCount(), static_cast<size_t>(0));

    // Clear all buffer metadata
    chunk_buffer_map[chunk_key]->resetToEmpty();
  }
  return chunk_buffer_map;
}

Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const

inline

Definition at line 92 of file ForeignStorageCache.h.

{ return cached_chunks_.size(); }

size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const

inline

Definition at line 93 of file ForeignStorageCache.h.

{ return cached_metadata_.size(); }

size_t foreign_storage::ForeignStorageCache::getNumChunksAdded ( ) const

inline

Definition at line 94 of file ForeignStorageCache.h.

{ return num_chunks_added_; }

size_t foreign_storage::ForeignStorageCache::getNumMetadataAdded ( ) const

inline

Definition at line 95 of file ForeignStorageCache.h.

{ return num_metadata_added_; }

uint64_t foreign_storage::ForeignStorageCache::getSpaceReservedByTable ( int  db_id, int  tb_id  ) const

inline

Definition at line 113 of file ForeignStorageCache.h.

                                                                      {
    return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
  }

bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix

(

const ChunkKey &

chunk_prefix

)

const

Definition at line 240 of file ForeignStorageCache.cpp.

References cached_metadata_, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                        {
  read_lock lock(metadata_mutex_);
  // We don't use iterateOvermatchingPrefix() here because we want to exit early if
  // possible.
  ChunkKey upper_prefix(chunk_prefix);
  upper_prefix.push_back(std::numeric_limits<int>::max());
  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;
       ++meta_it) {
    return true;
  }
  return false;
}

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached

(

const ChunkKey &

chunk_key

)

const

Definition at line 116 of file ForeignStorageCache.cpp.

References cached_metadata_, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                          {
  read_lock lock(metadata_mutex_);
  return (cached_metadata_.find(chunk_key) != cached_metadata_.end());
}

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::recoverCacheForTable	(	ChunkMetadataVector &	meta_vec,
		const ChunkKey &	table_key
	)

Definition at line 121 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, is_table_key(), is_varlen_data_key(), and is_varlen_key().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                                          {
  write_lock lock(chunks_mutex_);
  CHECK(meta_vec.size() == 0);
  CHECK(is_table_key(table_key));

  caching_file_mgr_->getChunkMetadataVecForKeyPrefix(meta_vec, table_key);
  for (auto& [chunk_key, metadata] : meta_vec) {
    cached_metadata_.emplace(chunk_key);
    // If there is no page count then the chunk was metadata only and should not be
    // cached.
    if (const auto& buf = caching_file_mgr_->getBuffer(chunk_key); buf->pageCount() > 0) {
      cached_chunks_.emplace(chunk_key);
    }

    if (is_varlen_key(chunk_key)) {
      // Metadata is only available for the data chunk, but look for the index as well
      CHECK(is_varlen_data_key(chunk_key));
      ChunkKey index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
                                  chunk_key[CHUNK_KEY_TABLE_IDX],
                                  chunk_key[CHUNK_KEY_COLUMN_IDX],
                                  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                                  2};

      if (const auto& buf = caching_file_mgr_->getBuffer(index_chunk_key);
          buf->pageCount() > 0) {
        cached_chunks_.emplace(index_chunk_key);
      }
    }
  }
  return (meta_vec.size() > 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string &  base_path ) const

private

Definition at line 365 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

                                                                       {
  // check if base_path already exists, and if not create one
  boost::filesystem::path path(base_path);
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      throw std::runtime_error{
          "cache path \"" + base_path +
          "\" is not a directory.  Please specify a valid directory "
          "with --disk_cache_path=<path>, or use the default location."};
    }
  } else {  // data directory does not exist
    if (!boost::filesystem::create_directory(path)) {
      throw std::runtime_error{
          "could not create directory at cache path \"" + base_path +
          "\".  Please specify a valid directory location "
          "with --disk_cache_path=<path> or use the default location."};
    }
  }
}

Here is the caller graph for this function:

Member Data Documentation

std::set<ChunkKey> foreign_storage::ForeignStorageCache::cached_chunks_

private

Definition at line 130 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheTableChunks(), clear(), clearForTablePrefix(), deleteBufferIfExists(), dumpCachedChunkEntries(), eraseChunk(), evictChunkByIterator(), getCachedChunkIfExists(), getCachedChunksForKeyPrefix(), getChunkBuffersForCaching(), and recoverCacheForTable().

std::set<ChunkKey> foreign_storage::ForeignStorageCache::cached_metadata_

private

Definition at line 131 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), clear(), clearForTablePrefix(), deleteBufferIfExists(), dumpCachedMetadataEntries(), getCachedMetadataVecForKeyPrefix(), hasCachedMetadataForKeyPrefix(), isMetadataCached(), and recoverCacheForTable().

std::unique_ptr<File_Namespace::CachingFileMgr> foreign_storage::ForeignStorageCache::caching_file_mgr_

private

Definition at line 127 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), cacheTableChunks(), clear(), clearForTablePrefix(), deleteBufferIfExists(), eraseChunk(), evictChunkByIterator(), ForeignStorageCache(), getCachedChunkIfExists(), getCachedMetadataVecForKeyPrefix(), getChunkBufferForPrecaching(), getChunkBuffersForCaching(), and recoverCacheForTable().

mapd_shared_mutex foreign_storage::ForeignStorageCache::chunks_mutex_

mutableprivate

Definition at line 138 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), cacheTableChunks(), clear(), clearForTablePrefix(), deleteBufferIfExists(), evictThenEraseChunk(), getCachedChunkIfExists(), getCachedChunksForKeyPrefix(), getChunkBuffersForCaching(), and recoverCacheForTable().

mapd_shared_mutex foreign_storage::ForeignStorageCache::metadata_mutex_

mutableprivate

Definition at line 139 of file ForeignStorageCache.h.

Referenced by cacheChunk(), cacheMetadataVec(), clear(), clearForTablePrefix(), deleteBufferIfExists(), getCachedMetadataVecForKeyPrefix(), hasCachedMetadataForKeyPrefix(), and isMetadataCached().

size_t foreign_storage::ForeignStorageCache::num_chunks_added_

private

Definition at line 134 of file ForeignStorageCache.h.

Referenced by cacheChunk(), and cacheTableChunks().

size_t foreign_storage::ForeignStorageCache::num_metadata_added_

private

Definition at line 135 of file ForeignStorageCache.h.

Referenced by cacheMetadataVec().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.cpp