OmniSciDB  a667adc9c8
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::ForeignStorageCache Class Reference

#include <ForeignStorageCache.h>

Public Member Functions

 ForeignStorageCache (const DiskCacheConfig &config)
 
void cacheTableChunks (const std::vector< ChunkKey > &chunk_keys)
 
void cacheChunk (const ChunkKey &, AbstractBuffer *)
 
AbstractBuffergetCachedChunkIfExists (const ChunkKey &)
 
bool isMetadataCached (const ChunkKey &) const
 
void cacheMetadataVec (const ChunkMetadataVector &)
 
void getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const
 
bool hasCachedMetadataForKeyPrefix (const ChunkKey &) const
 
void clearForTablePrefix (const ChunkKey &)
 
void clear ()
 
std::vector< ChunkKeygetCachedChunksForKeyPrefix (const ChunkKey &) const
 
bool recoverCacheForTable (ChunkMetadataVector &, const ChunkKey &)
 
ChunkToBufferMap getChunkBuffersForCaching (const std::vector< ChunkKey > &chunk_keys) const
 
AbstractBuffergetChunkBufferForPrecaching (const ChunkKey &chunk_key, bool is_new_buffer)
 
void deleteBufferIfExists (const ChunkKey &chunk_key)
 
size_t getNumCachedChunks () const
 
size_t getNumCachedMetadata () const
 
size_t getNumChunksAdded () const
 
size_t getNumMetadataAdded () const
 
std::string dumpCachedChunkEntries () const
 
std::string dumpCachedMetadataEntries () const
 
std::string getCacheDirectory () const
 
std::string getCacheDirectoryForTable (int db_id, int tb_id) const
 
void cacheMetadataWithFragIdGreaterOrEqualTo (const ChunkMetadataVector &metadata_vec, const int frag_id)
 
void evictThenEraseChunk (const ChunkKey &)
 
uint64_t getSpaceReservedByTable (int db_id, int tb_id) const
 

Private Member Functions

std::set< ChunkKey >::iterator eraseChunk (const std::set< ChunkKey >::iterator &)
 
void eraseChunk (const ChunkKey &chunk_key)
 
std::set< ChunkKey >::iterator evictChunkByIterator (const std::set< ChunkKey >::iterator &chunk_it)
 
void evictThenEraseChunkUnlocked (const ChunkKey &)
 
void validatePath (const std::string &) const
 

Private Attributes

std::unique_ptr
< File_Namespace::CachingFileMgr
caching_file_mgr_
 
std::set< ChunkKeycached_chunks_
 
std::set< ChunkKeycached_metadata_
 
size_t num_chunks_added_
 
size_t num_metadata_added_
 
mapd_shared_mutex chunks_mutex_
 
mapd_shared_mutex metadata_mutex_
 

Detailed Description

Definition at line 59 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache ( const DiskCacheConfig config)

Definition at line 54 of file ForeignStorageCache.cpp.

References caching_file_mgr_, DiskCacheConfig::num_reader_threads, DiskCacheConfig::path, and validatePath().

56  validatePath(config.path);
57  caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(
58  config.path, config.num_reader_threads);
59 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
void validatePath(const std::string &) const

+ Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheChunk ( const ChunkKey chunk_key,
AbstractBuffer buffer 
)

Definition at line 71 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, chunks_mutex_, Data_Namespace::AbstractBuffer::isDirty(), metadata_mutex_, num_chunks_added_, and Data_Namespace::AbstractBuffer::setUpdated().

71  {
72  write_lock meta_lock(metadata_mutex_);
73  write_lock chunk_lock(chunks_mutex_);
74  // We should only be caching buffers that are in sync with storage.
75  CHECK(!buffer->isDirty());
76  buffer->setUpdated();
78  caching_file_mgr_->putBuffer(chunk_key, buffer);
79  caching_file_mgr_->checkpoint();
80  cached_metadata_.emplace(chunk_key);
81  cached_chunks_.emplace(chunk_key);
82  CHECK(!buffer->isDirty());
83 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataVec ( const ChunkMetadataVector metadata_vec)

Definition at line 164 of file ForeignStorageCache.cpp.

References cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, evictThenEraseChunkUnlocked(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), is_varlen_data_key(), is_varlen_key(), metadata_mutex_, num_metadata_added_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by cacheMetadataWithFragIdGreaterOrEqualTo(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

164  {
165  auto timer = DEBUG_TIMER(__func__);
166  write_lock meta_lock(metadata_mutex_);
167  write_lock chunk_lock(chunks_mutex_);
168  for (auto& [chunk_key, metadata] : metadata_vec) {
169  cached_metadata_.emplace(chunk_key);
170  AbstractBuffer* buf;
171  AbstractBuffer* index_buffer = nullptr;
172  ChunkKey index_chunk_key;
173  if (is_varlen_key(chunk_key)) {
174  // For variable length chunks, metadata is associated with the data chunk.
175  CHECK(is_varlen_data_key(chunk_key));
176  index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
177  chunk_key[CHUNK_KEY_TABLE_IDX],
178  chunk_key[CHUNK_KEY_COLUMN_IDX],
179  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
180  2};
181  }
182  bool chunk_in_cache = false;
183  if (!caching_file_mgr_->isBufferOnDevice(chunk_key)) {
184  buf = caching_file_mgr_->createBuffer(chunk_key);
185 
186  if (!index_chunk_key.empty()) {
187  CHECK(!caching_file_mgr_->isBufferOnDevice(index_chunk_key));
188  index_buffer = caching_file_mgr_->createBuffer(index_chunk_key);
189  CHECK(index_buffer);
190  }
191  } else {
192  buf = caching_file_mgr_->getBuffer(chunk_key);
193 
194  if (!index_chunk_key.empty()) {
195  CHECK(caching_file_mgr_->isBufferOnDevice(index_chunk_key));
196  index_buffer = caching_file_mgr_->getBuffer(index_chunk_key);
197  CHECK(index_buffer);
198  }
199 
200  // We should have already cleared the data unless we are appending
201  // If the buffer metadata has changed, we need to remove this chunk
202  if (buf->getEncoder() != nullptr) {
203  const std::shared_ptr<ChunkMetadata> buf_metadata =
204  std::make_shared<ChunkMetadata>();
205  buf->getEncoder()->getMetadata(buf_metadata);
206  chunk_in_cache = *metadata.get() == *buf_metadata;
207  }
208  }
209 
210  if (!chunk_in_cache) {
211  set_metadata_for_buffer(buf, metadata.get());
212  evictThenEraseChunkUnlocked(chunk_key);
213 
214  if (!index_chunk_key.empty()) {
215  CHECK(index_buffer);
216  index_buffer->setUpdated();
217  evictThenEraseChunkUnlocked(index_chunk_key);
218  }
219  }
221  }
222  caching_file_mgr_->checkpoint();
223 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:70
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:227
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
An AbstractBuffer is a unit of data management for a data manager.
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
void evictThenEraseChunkUnlocked(const ChunkKey &)
void set_metadata_for_buffer(AbstractBuffer *buffer, ChunkMetadata *meta)
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:66

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataWithFragIdGreaterOrEqualTo ( const ChunkMetadataVector metadata_vec,
const int  frag_id 
)

Definition at line 385 of file ForeignStorageCache.cpp.

References cacheMetadataVec(), and CHUNK_KEY_FRAGMENT_IDX.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache().

387  {
388  // Only re-cache last fragment and above
389  ChunkMetadataVector new_metadata_vec;
390  for (const auto& chunk_metadata : metadata_vec) {
391  if (chunk_metadata.first[CHUNK_KEY_FRAGMENT_IDX] >= frag_id) {
392  new_metadata_vec.push_back(chunk_metadata);
393  }
394  }
395  cacheMetadataVec(new_metadata_vec);
396 }
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void cacheMetadataVec(const ChunkMetadataVector &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheTableChunks ( const std::vector< ChunkKey > &  chunk_keys)

Caches the chunks for the given chunk keys. Chunk buffers for chunks to be cached are expected to have already been populated before calling this method. This method also expects all provided chunk keys to be for the same table.

Parameters
chunk_keys- keys of chunks to be cached

Definition at line 85 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, CHECK, CHECK_EQ, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, and num_chunks_added_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

85  {
86  auto timer = DEBUG_TIMER(__func__);
88  CHECK(!chunk_keys.empty());
89 
90  auto db_id = chunk_keys[0][CHUNK_KEY_DB_IDX];
91  auto table_id = chunk_keys[0][CHUNK_KEY_TABLE_IDX];
92  const ChunkKey table_key{db_id, table_id};
93 
94  for (const auto& chunk_key : chunk_keys) {
95  CHECK_EQ(db_id, chunk_key[CHUNK_KEY_DB_IDX]);
96  CHECK_EQ(table_id, chunk_key[CHUNK_KEY_TABLE_IDX]);
97  CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
99  cached_chunks_.emplace(chunk_key);
100  }
101  caching_file_mgr_->checkpoint();
102 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::vector< int > ChunkKey
Definition: types.h:37
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear ( )

Definition at line 279 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), and metadata_mutex_.

279  {
280  auto timer = DEBUG_TIMER(__func__);
281  std::set<ChunkKey> table_keys;
282  {
283  write_lock w_lock(chunks_mutex_);
284  for (auto chunk_it = cached_chunks_.begin(); chunk_it != cached_chunks_.end();) {
285  chunk_it = evictChunkByIterator(chunk_it);
286  }
287  }
288  {
289  write_lock w_lock(metadata_mutex_);
290  for (auto meta_it = cached_metadata_.begin(); meta_it != cached_metadata_.end();) {
291  table_keys.emplace(ChunkKey{(*meta_it)[0], (*meta_it)[1]});
292  meta_it = cached_metadata_.erase(meta_it);
293  }
294  }
295  // FileMgrs do not clean up after themselves nicely, so we need to close all their disk
296  // resources and then re-create the CachingFileMgr to reset it.
297  caching_file_mgr_->closeRemovePhysical();
298  boost::filesystem::create_directory(caching_file_mgr_->getFileMgrBasePath());
299  caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(
300  caching_file_mgr_->getFileMgrBasePath(), caching_file_mgr_->getNumReaderThreads());
301 }
std::vector< int > ChunkKey
Definition: types.h:37
std::set< ChunkKey >::iterator evictChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::clearForTablePrefix ( const ChunkKey chunk_prefix)

Definition at line 255 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), is_table_key(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache(), and foreign_storage::CachingForeignStorageMgr::refreshTable().

255  {
256  CHECK(is_table_key(chunk_prefix));
257  auto timer = DEBUG_TIMER(__func__);
258  ChunkKey upper_prefix(chunk_prefix);
259  upper_prefix.push_back(std::numeric_limits<int>::max());
260  {
261  write_lock w_lock(chunks_mutex_);
262  // Delete chunks for prefix
263  auto end_it = cached_chunks_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
264  for (auto chunk_it = cached_chunks_.lower_bound(chunk_prefix); chunk_it != end_it;) {
265  chunk_it = evictChunkByIterator(chunk_it);
266  }
267  }
268  {
269  write_lock w_lock(metadata_mutex_);
270  // Delete metadata for prefix
271  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
272  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;) {
273  meta_it = cached_metadata_.erase(meta_it);
274  }
275  }
276  caching_file_mgr_->clearForTable(chunk_prefix[0], chunk_prefix[1]);
277 }
std::vector< int > ChunkKey
Definition: types.h:37
std::set< ChunkKey >::iterator evictChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists ( const ChunkKey chunk_key)

Definition at line 61 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, chunks_mutex_, and metadata_mutex_.

61  {
62  write_lock meta_lock(metadata_mutex_);
63  write_lock chunk_lock(chunks_mutex_);
64  if (cached_metadata_.find(chunk_key) != cached_metadata_.end()) {
65  caching_file_mgr_->deleteBuffer(chunk_key);
66  cached_chunks_.erase(chunk_key);
67  cached_metadata_.erase(chunk_key);
68  }
69 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_unique_lock< mapd_shared_mutex > write_lock
std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries ( ) const

Definition at line 349 of file ForeignStorageCache.cpp.

References cached_chunks_, and show_chunk().

349  {
350  std::string ret_string = "Cached chunks:\n";
351  for (const auto& chunk_key : cached_chunks_) {
352  ret_string += " " + show_chunk(chunk_key) + "\n";
353  }
354  return ret_string;
355 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:85

+ Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries ( ) const

Definition at line 357 of file ForeignStorageCache.cpp.

References cached_metadata_, and show_chunk().

357  {
358  std::string ret_string = "Cached ChunkMetadata:\n";
359  for (const auto& meta_key : cached_metadata_) {
360  ret_string += " " + show_chunk(meta_key) + "\n";
361  }
362  return ret_string;
363 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:85

+ Here is the call graph for this function:

std::set<ChunkKey>::iterator foreign_storage::ForeignStorageCache::eraseChunk ( const std::set< ChunkKey >::iterator &  )
private

Referenced by evictThenEraseChunkUnlocked().

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey chunk_key)
private

Definition at line 331 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and File_Namespace::FileBuffer::freeChunkPages().

331  {
332  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
333  return;
334  }
335  File_Namespace::FileBuffer* file_buffer =
336  static_cast<File_Namespace::FileBuffer*>(caching_file_mgr_->getBuffer(chunk_key));
337  file_buffer->freeChunkPages();
338  cached_chunks_.erase(chunk_key);
339 }
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:57
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

+ Here is the call graph for this function:

std::set< ChunkKey >::iterator foreign_storage::ForeignStorageCache::evictChunkByIterator ( const std::set< ChunkKey >::iterator &  chunk_it)
private

Definition at line 341 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and File_Namespace::FileBuffer::freeChunkPages().

Referenced by clear(), and clearForTablePrefix().

342  {
343  File_Namespace::FileBuffer* file_buffer =
344  static_cast<File_Namespace::FileBuffer*>(caching_file_mgr_->getBuffer(*chunk_it));
345  file_buffer->freeChunkPages();
346  return cached_chunks_.erase(chunk_it);
347 }
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:57
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunk ( const ChunkKey chunk_key)

Definition at line 154 of file ForeignStorageCache.cpp.

References chunks_mutex_, and evictThenEraseChunkUnlocked().

154  {
155  write_lock chunk_lock(chunks_mutex_);
156  evictThenEraseChunkUnlocked(chunk_key);
157 }
void evictThenEraseChunkUnlocked(const ChunkKey &)
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunkUnlocked ( const ChunkKey chunk_key)
private

Definition at line 159 of file ForeignStorageCache.cpp.

References eraseChunk(), and get_table_key().

Referenced by cacheMetadataVec(), and evictThenEraseChunk().

159  {
160  const ChunkKey table_prefix = get_table_key(chunk_key);
161  eraseChunk(chunk_key);
162 }
std::vector< int > ChunkKey
Definition: types.h:37
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists ( const ChunkKey chunk_key)

Definition at line 104 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and chunks_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

104  {
105  {
106  read_lock lock(chunks_mutex_);
107  // We do this instead of calling getBuffer so that we don't create a fileMgr if the
108  // chunk doesn't exist.
109  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
110  return nullptr;
111  }
112  }
113  return caching_file_mgr_->getBuffer(chunk_key);
114 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the caller graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 303 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

304  {
305  read_lock r_lock(chunks_mutex_);
306  std::vector<ChunkKey> ret_vec;
308  [&ret_vec](auto chunk) { ret_vec.push_back(chunk); }, cached_chunks_, chunk_prefix);
309  return ret_vec;
310 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectory ( ) const
inline

Definition at line 101 of file ForeignStorageCache.h.

101  {
102  return caching_file_mgr_->getFileMgrBasePath();
103  }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTable ( int  db_id,
int  tb_id 
) const
inline

Definition at line 105 of file ForeignStorageCache.h.

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk().

105  {
106  return caching_file_mgr_->getOrAddTableDir(db_id, tb_id);
107  }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix ( ChunkMetadataVector metadata_vec,
const ChunkKey chunk_prefix 
) const

Definition at line 225 of file ForeignStorageCache.cpp.

References cached_metadata_, caching_file_mgr_, DEBUG_TIMER, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

227  {
228  auto timer = DEBUG_TIMER(__func__);
229  read_lock r_lock(metadata_mutex_);
231  [&metadata_vec, this](auto chunk) {
232  std::shared_ptr<ChunkMetadata> buf_metadata = std::make_shared<ChunkMetadata>();
233  caching_file_mgr_->getBuffer(chunk)->getEncoder()->getMetadata(buf_metadata);
234  metadata_vec.push_back(std::make_pair(chunk, buf_metadata));
235  },
237  chunk_prefix);
238 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getChunkBufferForPrecaching ( const ChunkKey chunk_key,
bool  is_new_buffer 
)

Definition at line 398 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and CHECK.

400  {
401  if (!is_new_buffer) {
402  CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
403  return caching_file_mgr_->getBuffer(chunk_key);
404  } else {
405  CHECK(!caching_file_mgr_->isBufferOnDevice(chunk_key));
406  return caching_file_mgr_->createBuffer(chunk_key);
407  }
408 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
ChunkToBufferMap foreign_storage::ForeignStorageCache::getChunkBuffersForCaching ( const std::vector< ChunkKey > &  chunk_keys) const

Definition at line 312 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, CHECK, CHECK_EQ, and chunks_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

313  {
314  ChunkToBufferMap chunk_buffer_map;
315  read_lock lock(chunks_mutex_);
316  for (const auto& chunk_key : chunk_keys) {
317  CHECK(cached_chunks_.find(chunk_key) == cached_chunks_.end());
318  CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
319  chunk_buffer_map[chunk_key] = caching_file_mgr_->getBuffer(chunk_key);
320  CHECK(dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[chunk_key]));
321  CHECK_EQ(chunk_buffer_map[chunk_key]->pageCount(), static_cast<size_t>(0));
322 
323  // Clear all buffer metadata
324  chunk_buffer_map[chunk_key]->resetToEmpty();
325  }
326  return chunk_buffer_map;
327 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const
inline

Definition at line 92 of file ForeignStorageCache.h.

92 { return cached_chunks_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const
inline

Definition at line 93 of file ForeignStorageCache.h.

93 { return cached_metadata_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumChunksAdded ( ) const
inline

Definition at line 94 of file ForeignStorageCache.h.

size_t foreign_storage::ForeignStorageCache::getNumMetadataAdded ( ) const
inline

Definition at line 95 of file ForeignStorageCache.h.

uint64_t foreign_storage::ForeignStorageCache::getSpaceReservedByTable ( int  db_id,
int  tb_id 
) const
inline

Definition at line 113 of file ForeignStorageCache.h.

113  {
114  return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
115  }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 240 of file ForeignStorageCache.cpp.

References cached_metadata_, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

241  {
243  // We don't use iterateOvermatchingPrefix() here because we want to exit early if
244  // possible.
245  ChunkKey upper_prefix(chunk_prefix);
246  upper_prefix.push_back(std::numeric_limits<int>::max());
247  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
248  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;
249  ++meta_it) {
250  return true;
251  }
252  return false;
253 }
std::vector< int > ChunkKey
Definition: types.h:37
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached ( const ChunkKey chunk_key) const

Definition at line 116 of file ForeignStorageCache.cpp.

References cached_metadata_, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

116  {
118  return (cached_metadata_.find(chunk_key) != cached_metadata_.end());
119 }
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::recoverCacheForTable ( ChunkMetadataVector meta_vec,
const ChunkKey table_key 
)

Definition at line 121 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, is_table_key(), is_varlen_data_key(), and is_varlen_key().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

122  {
124  CHECK(meta_vec.size() == 0);
125  CHECK(is_table_key(table_key));
126 
127  caching_file_mgr_->getChunkMetadataVecForKeyPrefix(meta_vec, table_key);
128  for (auto& [chunk_key, metadata] : meta_vec) {
129  cached_metadata_.emplace(chunk_key);
130  // If there is no page count then the chunk was metadata only and should not be
131  // cached.
132  if (const auto& buf = caching_file_mgr_->getBuffer(chunk_key); buf->pageCount() > 0) {
133  cached_chunks_.emplace(chunk_key);
134  }
135 
136  if (is_varlen_key(chunk_key)) {
137  // Metadata is only available for the data chunk, but look for the index as well
138  CHECK(is_varlen_data_key(chunk_key));
139  ChunkKey index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
140  chunk_key[CHUNK_KEY_TABLE_IDX],
141  chunk_key[CHUNK_KEY_COLUMN_IDX],
142  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
143  2};
144 
145  if (const auto& buf = caching_file_mgr_->getBuffer(index_chunk_key);
146  buf->pageCount() > 0) {
147  cached_chunks_.emplace(index_chunk_key);
148  }
149  }
150  }
151  return (meta_vec.size() > 0);
152 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:70
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:66

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string &  base_path) const
private

Definition at line 365 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

365  {
366  // check if base_path already exists, and if not create one
367  boost::filesystem::path path(base_path);
368  if (boost::filesystem::exists(path)) {
369  if (!boost::filesystem::is_directory(path)) {
370  throw std::runtime_error{
371  "cache path \"" + base_path +
372  "\" is not a directory. Please specify a valid directory "
373  "with --disk_cache_path=<path>, or use the default location."};
374  }
375  } else { // data directory does not exist
376  if (!boost::filesystem::create_directory(path)) {
377  throw std::runtime_error{
378  "could not create directory at cache path \"" + base_path +
379  "\". Please specify a valid directory location "
380  "with --disk_cache_path=<path> or use the default location."};
381  }
382  }
383 }

+ Here is the caller graph for this function:

Member Data Documentation

size_t foreign_storage::ForeignStorageCache::num_chunks_added_
private

Definition at line 134 of file ForeignStorageCache.h.

Referenced by cacheChunk(), and cacheTableChunks().

size_t foreign_storage::ForeignStorageCache::num_metadata_added_
private

Definition at line 135 of file ForeignStorageCache.h.

Referenced by cacheMetadataVec().


The documentation for this class was generated from the following files: