foreign_storage::ForeignStorageCache Class Reference

#include <ForeignStorageCache.h>

Public Member Functions
	ForeignStorageCache (const File_Namespace::DiskCacheConfig &config)
void	checkpoint (const int32_t db_id, const int32_t tb_id)
void	putBuffer (const ChunkKey &, AbstractBuffer *, const size_t numBytes=0)
File_Namespace::FileBuffer *	getCachedChunkIfExists (const ChunkKey &)
bool	isMetadataCached (const ChunkKey &) const
void	cacheMetadataVec (const ChunkMetadataVector &)
void	getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const
bool	hasCachedMetadataForKeyPrefix (const ChunkKey &) const
void	clearForTablePrefix (const ChunkKey &)
void	clear ()
size_t	getMaxChunkDataSize () const
std::vector< ChunkKey >	getCachedChunksForKeyPrefix (const ChunkKey &) const
ChunkToBufferMap	getChunkBuffersForCaching (const std::set< ChunkKey > &chunk_keys) const
AbstractBuffer *	getChunkBufferForPrecaching (const ChunkKey &chunk_key, bool is_new_buffer)
void	deleteBufferIfExists (const ChunkKey &chunk_key)
size_t	getNumCachedChunks () const
size_t	getNumCachedMetadata () const
std::string	dumpCachedChunkEntries () const
std::string	dumpCachedMetadataEntries () const
std::string	dumpEvictionQueue () const
std::string	dump () const
std::string	getCacheDirectory () const
std::string	getCacheDirectoryForTable (int db_id, int tb_id) const
std::string	getSerializedWrapperPath (int32_t db_id, int32_t tb_id) const
uint64_t	getSpaceReservedByTable (int db_id, int tb_id) const
void	storeDataWrapper (const std::string &doc, int32_t db_id, int32_t tb_id)
bool	hasStoredDataWrapperMetadata (int32_t db_id, int32_t table_id) const
void	setDataSizeLimit (size_t max) const

Private Member Functions
std::set< ChunkKey >::iterator	eraseChunk (const std::set< ChunkKey >::iterator &)
void	eraseChunk (const ChunkKey &chunk_key)
void	validatePath (const std::string &) const

Private Attributes
std::unique_ptr < File_Namespace::CachingFileMgr >	caching_file_mgr_

Detailed Description

Definition at line 42 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache

(

const File_Namespace::DiskCacheConfig &

config

)

Definition at line 55 of file ForeignStorageCache.cpp.

References caching_file_mgr_, File_Namespace::DiskCacheConfig::path, and validatePath().

                                                                                    {
  validatePath(config.path);
  caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(config);
}

Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheMetadataVec

(

const ChunkMetadataVector &

metadata_vec

)

Definition at line 112 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, DEBUG_TIMER, eraseChunk(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), in_same_table(), is_varlen_data_key(), is_varlen_key(), foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix(), foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

                                                                                  {
  auto timer = DEBUG_TIMER(__func__);
  if (metadata_vec.empty()) {
    return;
  }
  auto first_chunk_key = metadata_vec.begin()->first;
  for (auto& [chunk_key, metadata] : metadata_vec) {
    CHECK(in_same_table(chunk_key, first_chunk_key));
    AbstractBuffer* buf;
    AbstractBuffer* index_buffer = nullptr;
    ChunkKey index_chunk_key;
    if (is_varlen_key(chunk_key)) {
      // For variable length chunks, metadata is associated with the data chunk.
      CHECK(is_varlen_data_key(chunk_key));
      index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
                         chunk_key[CHUNK_KEY_TABLE_IDX],
                         chunk_key[CHUNK_KEY_COLUMN_IDX],
                         chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                         2};
    }
    bool chunk_in_cache = false;
    if (!caching_file_mgr_->isBufferOnDevice(chunk_key)) {
      buf = caching_file_mgr_->createBuffer(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(!caching_file_mgr_->isBufferOnDevice(index_chunk_key));
        index_buffer = caching_file_mgr_->createBuffer(index_chunk_key);
        CHECK(index_buffer);
      }
    } else {
      buf = caching_file_mgr_->getBuffer(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(caching_file_mgr_->isBufferOnDevice(index_chunk_key));
        index_buffer = caching_file_mgr_->getBuffer(index_chunk_key);
        CHECK(index_buffer);
      }

      // We should have already cleared the data unless we are appending
      // If the buffer metadata has changed, we need to remove this chunk
      if (buf->getEncoder() != nullptr) {
        const std::shared_ptr<ChunkMetadata> buf_metadata =
            std::make_shared<ChunkMetadata>();
        buf->getEncoder()->getMetadata(buf_metadata);
        chunk_in_cache = *metadata.get() == *buf_metadata;
      }
    }

    if (!chunk_in_cache) {
      set_metadata_for_buffer(buf, metadata.get());
      eraseChunk(chunk_key);

      if (!index_chunk_key.empty()) {
        CHECK(index_buffer);
        index_buffer->setUpdated();
        eraseChunk(index_chunk_key);
      }
    }
  }
  caching_file_mgr_->checkpoint(first_chunk_key[CHUNK_KEY_DB_IDX],
                                first_chunk_key[CHUNK_KEY_TABLE_IDX]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::checkpoint	(	const int32_t	db_id,
		const int32_t	tb_id
	)

Definition at line 71 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper(), and foreign_storage::CachingForeignStorageMgr::populateChunkBuffersSafely().

                                                                             {
  caching_file_mgr_->checkpoint(db_id, tb_id);
}

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear

(

)

Definition at line 193 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and DEBUG_TIMER.

                                {
  auto timer = DEBUG_TIMER(__func__);
  // FileMgrs do not clean up after themselves nicely, so we need to close all their disk
  // resources and then re-create the CachingFileMgr to reset it.
  caching_file_mgr_->closeRemovePhysical();
  boost::filesystem::create_directory(caching_file_mgr_->getFileMgrBasePath());
  caching_file_mgr_ = caching_file_mgr_->reconstruct();
}

void foreign_storage::ForeignStorageCache::clearForTablePrefix

(

const ChunkKey &

chunk_prefix

)

Definition at line 186 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, DEBUG_TIMER, and is_table_key().

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::clear_cached_table_data(), foreign_storage::CachingForeignStorageMgr::clearTable(), File_Namespace::CachingGlobalFileMgr::deleteBuffersWithPrefix(), foreign_storage::CachingForeignStorageMgr::removeTableRelatedDS(), and File_Namespace::CachingGlobalFileMgr::removeTableRelatedDS().

                                                                          {
  CHECK(is_table_key(chunk_prefix));
  auto timer = DEBUG_TIMER(__func__);
  caching_file_mgr_->clearForTable(chunk_prefix[CHUNK_KEY_DB_IDX],
                                   chunk_prefix[CHUNK_KEY_TABLE_IDX]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists

(

const ChunkKey &

chunk_key

)

Definition at line 60 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by File_Namespace::CachingGlobalFileMgr::deleteBuffer().

                                                                        {
  caching_file_mgr_->deleteBufferIfExists(chunk_key);
}

Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::dump ( ) const

inline

Definition at line 77 of file ForeignStorageCache.h.

{ return caching_file_mgr_->dump(); }

std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries

(

)

const

Definition at line 211 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

                                                            {
  return caching_file_mgr_->dumpKeysWithChunkData();
}

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries

(

)

const

Definition at line 215 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

                                                               {
  return caching_file_mgr_->dumpKeysWithMetadata();
}

std::string foreign_storage::ForeignStorageCache::dumpEvictionQueue

(

)

const

Definition at line 219 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

                                                       {
  return caching_file_mgr_->dumpEvictionQueue();
}

std::set<ChunkKey>::iterator foreign_storage::ForeignStorageCache::eraseChunk ( const std::set< ChunkKey >::iterator &  )

private

Referenced by cacheMetadataVec().

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey &  chunk_key )

private

Definition at line 207 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

                                                              {
  caching_file_mgr_->removeChunkKeepMetadata(chunk_key);
}

File_Namespace::FileBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists

(

const ChunkKey &

chunk_key

)

Definition at line 75 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHUNK_KEY_VARLEN_IDX, and is_varlen_data_key().

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                               {
  auto buf = caching_file_mgr_->getBufferIfExists(chunk_key);

  if (buf) {
    if ((*buf)->hasDataPages()) {
      // 1. If the buffer has data pages then must be in the cache.
      return *buf;
    }
    if (is_varlen_data_key(chunk_key)) {
      // 2. If the buffer is a varlen data buffer and the
      // corresponding chunk contains only nulls, then even
      // without data pages it will still have been cached
      // if it has a corresponding index buffer which does
      // have dataPages
      // Note the empty buffer proviso that the size be 0,
      // corresponding to  all nulls in the chunk
      auto index_chunk_key = chunk_key;
      index_chunk_key[CHUNK_KEY_VARLEN_IDX] = 2;
      auto index_buffer = caching_file_mgr_->getBufferIfExists(index_chunk_key);
      if (index_buffer && (*index_buffer)->hasDataPages() && (*buf)->size() == 0) {
        return *buf;
      }
    }
  }
  // 3. Otherwise this chunk hasn't been cached.
  return nullptr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix

(

const ChunkKey &

chunk_prefix

)

const

Definition at line 202 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                        {
  return caching_file_mgr_->getChunkKeysForPrefix(chunk_prefix);
}

Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectory ( ) const

inline

Definition at line 79 of file ForeignStorageCache.h.

                                             {
    return caching_file_mgr_->getFileMgrBasePath();
  }

std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTable ( int  db_id, int  tb_id  ) const

inline

Definition at line 83 of file ForeignStorageCache.h.

                                                                         {
    return caching_file_mgr_->getTableFileMgrPath(db_id, tb_id);
  }

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix	(	ChunkMetadataVector &	metadata_vec,
		const ChunkKey &	chunk_prefix
	)		const

Definition at line 175 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::createDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getBufferSize(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

                                        {
  caching_file_mgr_->getChunkMetadataVecForKeyPrefix(metadata_vec, chunk_prefix);
}

Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getChunkBufferForPrecaching	(	const ChunkKey &	chunk_key,
		bool	is_new_buffer
	)

Definition at line 259 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and CHECK.

                        {
  if (!is_new_buffer) {
    CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
    return caching_file_mgr_->getBuffer(chunk_key);
  } else {
    CHECK(!caching_file_mgr_->isBufferOnDevice(chunk_key));
    return caching_file_mgr_->createBuffer(chunk_key);
  }
}

ChunkToBufferMap foreign_storage::ForeignStorageCache::getChunkBuffersForCaching

(

const std::set< ChunkKey > &

chunk_keys

)

const

Definition at line 243 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and CHECK.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                        {
  ChunkToBufferMap chunk_buffer_map;
  for (const auto& key : keys) {
    CHECK(caching_file_mgr_->isBufferOnDevice(key));
    chunk_buffer_map[key] = caching_file_mgr_->getBuffer(key);
    auto file_buf = dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[key]);
    CHECK(file_buf);
    CHECK(!file_buf->hasDataPages());

    // Clear all buffer metadata
    file_buf->resetToEmpty();
  }
  return chunk_buffer_map;
}

Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getMaxChunkDataSize ( ) const

inline

Definition at line 55 of file ForeignStorageCache.h.

Referenced by foreign_storage::CachingForeignStorageMgr::maxFetchSize().

{ return caching_file_mgr_->getMaxDataFilesSize(); }

Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const

inline

Definition at line 66 of file ForeignStorageCache.h.

                                           {
    return caching_file_mgr_->getNumDataChunks();
  }

size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const

inline

Definition at line 69 of file ForeignStorageCache.h.

                                             {
    return caching_file_mgr_->getNumChunksWithMetadata();
  }

std::string foreign_storage::ForeignStorageCache::getSerializedWrapperPath ( int32_t  db_id, int32_t  tb_id  ) const

inline

Definition at line 87 of file ForeignStorageCache.h.

References File_Namespace::CachingFileMgr::WRAPPER_FILE_NAME.

Referenced by foreign_storage::CachingForeignStorageMgr::createDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::eraseDataWrapper().

                                                                                {
    return getCacheDirectoryForTable(db_id, tb_id) + "/" +
           File_Namespace::CachingFileMgr::WRAPPER_FILE_NAME;
  }

Here is the caller graph for this function:

uint64_t foreign_storage::ForeignStorageCache::getSpaceReservedByTable ( int  db_id, int  tb_id  ) const

inline

Definition at line 92 of file ForeignStorageCache.h.

                                                                      {
    return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
  }

bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix

(

const ChunkKey &

chunk_prefix

)

const

Definition at line 181 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::clearTable(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

                                        {
  return caching_file_mgr_->hasChunkMetadataForKeyPrefix(chunk_prefix);
}

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::hasStoredDataWrapperMetadata	(	int32_t	db_id,
		int32_t	table_id
	)		const

Definition at line 277 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::hasStoredDataWrapper().

                                                                               {
  return caching_file_mgr_->hasWrapperFile(db_id, table_id);
}

Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached

(

const ChunkKey &

chunk_key

)

const

Definition at line 104 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

                                                                          {
  auto buf = caching_file_mgr_->getBufferIfExists(chunk_key);
  if (buf) {
    return (*buf)->hasEncoder();
  }
  return false;
}

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::putBuffer	(	const ChunkKey &	key,
		AbstractBuffer *	buf,
		const size_t	numBytes = 0
	)

Definition at line 64 of file ForeignStorageCache.cpp.

References caching_file_mgr_, CHECK, and Data_Namespace::AbstractBuffer::isDirty().

Referenced by File_Namespace::CachingGlobalFileMgr::checkpoint(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), and File_Namespace::CachingGlobalFileMgr::putBuffer().

                                                            {
  caching_file_mgr_->putBuffer(key, buf, num_bytes);
  CHECK(!buf->isDirty());
}

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::setDataSizeLimit ( size_t  max ) const

inline

Definition at line 101 of file ForeignStorageCache.h.

                                                 {
    caching_file_mgr_->setDataSizeLimit(max);
  }

void foreign_storage::ForeignStorageCache::storeDataWrapper	(	const std::string &	doc,
		int32_t	db_id,
		int32_t	tb_id
	)

Definition at line 271 of file ForeignStorageCache.cpp.

References caching_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper().

                                                          {
  caching_file_mgr_->writeWrapperFile(doc, db_id, tb_id);
}

Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string &  base_path ) const

private

Definition at line 223 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

                                                                       {
  // check if base_path already exists, and if not create one
  boost::filesystem::path path(base_path);
  if (boost::filesystem::exists(path)) {
    if (!boost::filesystem::is_directory(path)) {
      throw std::runtime_error{
          "cache path \"" + base_path +
          "\" is not a directory.  Please specify a valid directory "
          "with --disk_cache_path=<path>, or use the default location."};
    }
  } else {  // data directory does not exist
    if (!boost::filesystem::create_directory(path)) {
      throw std::runtime_error{
          "could not create directory at cache path \"" + base_path +
          "\".  Please specify a valid directory location "
          "with --disk_cache_path=<path> or use the default location."};
    }
  }
}

Here is the caller graph for this function:

Member Data Documentation

std::unique_ptr<File_Namespace::CachingFileMgr> foreign_storage::ForeignStorageCache::caching_file_mgr_

private

Definition at line 112 of file ForeignStorageCache.h.

Referenced by cacheMetadataVec(), checkpoint(), clear(), clearForTablePrefix(), deleteBufferIfExists(), dumpCachedChunkEntries(), dumpCachedMetadataEntries(), dumpEvictionQueue(), eraseChunk(), ForeignStorageCache(), getCachedChunkIfExists(), getCachedChunksForKeyPrefix(), getCachedMetadataVecForKeyPrefix(), getChunkBufferForPrecaching(), getChunkBuffersForCaching(), hasCachedMetadataForKeyPrefix(), hasStoredDataWrapperMetadata(), isMetadataCached(), putBuffer(), and storeDataWrapper().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageCache.cpp