OmniSciDB  95562058bd
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::ForeignStorageCache Class Reference

#include <ForeignStorageCache.h>

Public Member Functions

 ForeignStorageCache (const DiskCacheConfig &config)
 
void cacheTableChunks (const std::vector< ChunkKey > &chunk_keys)
 
void cacheChunk (const ChunkKey &, AbstractBuffer *)
 
AbstractBuffergetCachedChunkIfExists (const ChunkKey &)
 
bool isMetadataCached (const ChunkKey &) const
 
void cacheMetadataVec (const ChunkMetadataVector &)
 
void getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const
 
bool hasCachedMetadataForKeyPrefix (const ChunkKey &) const
 
void clearForTablePrefix (const ChunkKey &)
 
void clear ()
 
void setLimit (uint64_t limit)
 
std::vector< ChunkKeygetCachedChunksForKeyPrefix (const ChunkKey &) const
 
bool recoverCacheForTable (ChunkMetadataVector &, const ChunkKey &)
 
std::map< ChunkKey,
AbstractBuffer * > 
getChunkBuffersForCaching (const std::vector< ChunkKey > &chunk_keys) const
 
void deleteBufferIfExists (const ChunkKey &chunk_key)
 
uint64_t getLimit () const
 
size_t getNumCachedChunks () const
 
size_t getNumCachedMetadata () const
 
size_t getNumChunksAdded () const
 
size_t getNumMetadataAdded () const
 
std::string dumpCachedChunkEntries () const
 
std::string dumpCachedMetadataEntries () const
 
std::string dumpEvictionQueue () const
 
File_Namespace::GlobalFileMgrgetGlobalFileMgr () const
 
std::string getCacheDirectoryForTablePrefix (const ChunkKey &) const
 
void cacheMetadataWithFragIdGreaterOrEqualTo (const ChunkMetadataVector &metadata_vec, const int frag_id)
 
void evictThenEraseChunk (const ChunkKey &)
 

Private Member Functions

std::set< ChunkKey >::iterator eraseChunk (const std::set< ChunkKey >::iterator &)
 
void eraseChunk (const ChunkKey &, TableEvictionTracker &tracker)
 
std::set< ChunkKey >::iterator eraseChunkByIterator (const std::set< ChunkKey >::iterator &chunk_it)
 
void evictThenEraseChunkUnlocked (const ChunkKey &)
 
void validatePath (const std::string &) const
 
bool insertChunkIntoEvictionAlg (const ChunkKey &, const size_t)
 
void createTrackerMapEntryIfNoneExists (const ChunkKey &chunk_key)
 

Private Attributes

std::map< const ChunkKey,
TableEvictionTracker
eviction_tracker_map_
 
uint64_t max_pages_per_table_
 
std::unique_ptr
< File_Namespace::GlobalFileMgr
global_file_mgr_
 
std::set< ChunkKeycached_chunks_
 
std::set< ChunkKeycached_metadata_
 
size_t num_chunks_added_
 
size_t num_metadata_added_
 
mapd_shared_mutex chunks_mutex_
 
mapd_shared_mutex metadata_mutex_
 
uint64_t max_cached_bytes_
 

Detailed Description

Definition at line 68 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache ( const DiskCacheConfig config)

Definition at line 54 of file ForeignStorageCache.cpp.

References global_file_mgr_, DiskCacheConfig::num_reader_threads, DiskCacheConfig::path, setLimit(), DiskCacheConfig::size_limit, and validatePath().

56  validatePath(config.path);
57  global_file_mgr_ = std::make_unique<File_Namespace::GlobalFileMgr>(
58  0, config.path, config.num_reader_threads);
59  setLimit(config.size_limit);
60 }
void validatePath(const std::string &) const
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_

+ Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheChunk ( const ChunkKey chunk_key,
AbstractBuffer buffer 
)

Definition at line 76 of file ForeignStorageCache.cpp.

References cached_metadata_, CHECK, chunks_mutex_, DEBUG_TIMER, global_file_mgr_, insertChunkIntoEvictionAlg(), Data_Namespace::AbstractBuffer::isDirty(), metadata_mutex_, num_chunks_added_, Data_Namespace::AbstractBuffer::setUpdated(), and Data_Namespace::AbstractBuffer::size().

76  {
77  auto timer = DEBUG_TIMER(__func__);
78  write_lock meta_lock(metadata_mutex_);
79  write_lock chunk_lock(chunks_mutex_);
80  // We should only be caching buffers that are in sync with storage.
81  CHECK(!buffer->isDirty());
82  if (insertChunkIntoEvictionAlg(chunk_key, buffer->size())) {
83  buffer->setUpdated();
85  global_file_mgr_->putBuffer(chunk_key, buffer);
86  global_file_mgr_->checkpoint();
87  // TODO(Misiu): This needs to happen even if insertChunkIntoEvictionAlg() fails.
88  cached_metadata_.emplace(chunk_key);
89  }
90  CHECK(!buffer->isDirty());
91 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
bool insertChunkIntoEvictionAlg(const ChunkKey &, const size_t)
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataVec ( const ChunkMetadataVector metadata_vec)

Definition at line 170 of file ForeignStorageCache.cpp.

References cached_metadata_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, evictThenEraseChunkUnlocked(), global_file_mgr_, is_varlen_data_key(), is_varlen_key(), metadata_mutex_, num_metadata_added_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by cacheMetadataWithFragIdGreaterOrEqualTo(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

170  {
171  auto timer = DEBUG_TIMER(__func__);
172  write_lock meta_lock(metadata_mutex_);
173  write_lock chunk_lock(chunks_mutex_);
174  for (auto& [chunk_key, metadata] : metadata_vec) {
175  cached_metadata_.emplace(chunk_key);
176  AbstractBuffer* buf;
177  AbstractBuffer* index_buffer = nullptr;
178  ChunkKey index_chunk_key;
179  if (is_varlen_key(chunk_key)) {
180  // For variable length chunks, metadata is associated with the data chunk
181  CHECK(is_varlen_data_key(chunk_key));
182  index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
183  chunk_key[CHUNK_KEY_TABLE_IDX],
184  chunk_key[CHUNK_KEY_COLUMN_IDX],
185  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
186  2};
187  }
188 
189  if (!global_file_mgr_->isBufferOnDevice(chunk_key)) {
190  buf = global_file_mgr_->createBuffer(chunk_key);
191 
192  if (!index_chunk_key.empty()) {
193  CHECK(!global_file_mgr_->isBufferOnDevice(index_chunk_key));
194  index_buffer = global_file_mgr_->createBuffer(index_chunk_key);
195  CHECK(index_buffer);
196  }
197  } else {
198  buf = global_file_mgr_->getBuffer(chunk_key);
199 
200  if (!index_chunk_key.empty()) {
201  CHECK(global_file_mgr_->isBufferOnDevice(index_chunk_key));
202  index_buffer = global_file_mgr_->getBuffer(index_chunk_key);
203  CHECK(index_buffer);
204  }
205  }
206 
207  set_metadata_for_buffer(buf, metadata.get());
208  evictThenEraseChunkUnlocked(chunk_key);
209 
210  if (!index_chunk_key.empty()) {
211  CHECK(index_buffer);
212  index_buffer->setUpdated();
213  evictThenEraseChunkUnlocked(index_chunk_key);
214  }
216  }
217  global_file_mgr_->checkpoint();
218 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:65
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
An AbstractBuffer is a unit of data management for a data manager.
void evictThenEraseChunkUnlocked(const ChunkKey &)
void set_metadata_for_buffer(AbstractBuffer *buffer, ChunkMetadata *meta)
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:61

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataWithFragIdGreaterOrEqualTo ( const ChunkMetadataVector metadata_vec,
const int  frag_id 
)

Definition at line 458 of file ForeignStorageCache.cpp.

References cacheMetadataVec(), and CHUNK_KEY_FRAGMENT_IDX.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache().

460  {
461  // Only re-cache last fragment and above
462  ChunkMetadataVector new_metadata_vec;
463  for (const auto& chunk_metadata : metadata_vec) {
464  if (chunk_metadata.first[CHUNK_KEY_FRAGMENT_IDX] >= frag_id) {
465  new_metadata_vec.push_back(chunk_metadata);
466  }
467  }
468  cacheMetadataVec(new_metadata_vec);
469 }
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void cacheMetadataVec(const ChunkMetadataVector &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheTableChunks ( const std::vector< ChunkKey > &  chunk_keys)

Caches the chunks for the given chunk keys. Chunk buffers for chunks to be cached are expected to have already been populated before calling this method. This method also expects all provided chunk keys to be for the same table.

Parameters
chunk_keys- keys of chunks to be cached

Definition at line 93 of file ForeignStorageCache.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, createTrackerMapEntryIfNoneExists(), DEBUG_TIMER, global_file_mgr_, insertChunkIntoEvictionAlg(), and num_chunks_added_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

93  {
94  auto timer = DEBUG_TIMER(__func__);
96  CHECK(!chunk_keys.empty());
97 
98  auto db_id = chunk_keys[0][CHUNK_KEY_DB_IDX];
99  auto table_id = chunk_keys[0][CHUNK_KEY_TABLE_IDX];
100  const ChunkKey table_key{db_id, table_id};
101 
103  for (const auto& chunk_key : chunk_keys) {
104  CHECK_EQ(db_id, chunk_key[CHUNK_KEY_DB_IDX]);
105  CHECK_EQ(table_id, chunk_key[CHUNK_KEY_TABLE_IDX]);
106  CHECK(global_file_mgr_->isBufferOnDevice(chunk_key));
108  insertChunkIntoEvictionAlg(chunk_key, global_file_mgr_->getBuffer(chunk_key)->size());
109  }
110  global_file_mgr_->checkpoint(db_id, table_id);
111 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::vector< int > ChunkKey
Definition: types.h:37
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
bool insertChunkIntoEvictionAlg(const ChunkKey &, const size_t)
void createTrackerMapEntryIfNoneExists(const ChunkKey &chunk_key)
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear ( )

Definition at line 275 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, chunks_mutex_, DEBUG_TIMER, eraseChunkByIterator(), global_file_mgr_, and metadata_mutex_.

275  {
276  auto timer = DEBUG_TIMER(__func__);
277  std::set<ChunkKey> table_keys;
278  {
279  write_lock w_lock(chunks_mutex_);
280  for (auto chunk_it = cached_chunks_.begin(); chunk_it != cached_chunks_.end();) {
281  chunk_it = eraseChunkByIterator(chunk_it);
282  }
283  }
284  {
285  write_lock w_lock(metadata_mutex_);
286  for (auto meta_it = cached_metadata_.begin(); meta_it != cached_metadata_.end();) {
287  table_keys.emplace(ChunkKey{(*meta_it)[0], (*meta_it)[1]});
288  meta_it = cached_metadata_.erase(meta_it);
289  }
290  }
291  for (const auto& table_key : table_keys) {
292  global_file_mgr_->removeTableRelatedDS(table_key[0], table_key[1]);
293  }
294 }
std::vector< int > ChunkKey
Definition: types.h:37
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
std::set< ChunkKey >::iterator eraseChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::clearForTablePrefix ( const ChunkKey chunk_prefix)

Definition at line 251 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, CHECK, chunks_mutex_, DEBUG_TIMER, eraseChunkByIterator(), global_file_mgr_, is_table_key(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache(), and foreign_storage::CachingForeignStorageMgr::refreshTable().

251  {
252  CHECK(is_table_key(chunk_prefix));
253  auto timer = DEBUG_TIMER(__func__);
254  ChunkKey upper_prefix(chunk_prefix);
255  upper_prefix.push_back(std::numeric_limits<int>::max());
256  {
257  write_lock w_lock(chunks_mutex_);
258  // Delete chunks for prefix
259  auto end_it = cached_chunks_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
260  for (auto chunk_it = cached_chunks_.lower_bound(chunk_prefix); chunk_it != end_it;) {
261  chunk_it = eraseChunkByIterator(chunk_it);
262  }
263  }
264  {
265  write_lock w_lock(metadata_mutex_);
266  // Delete metadata for prefix
267  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
268  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;) {
269  meta_it = cached_metadata_.erase(meta_it);
270  }
271  }
272  global_file_mgr_->removeTableRelatedDS(chunk_prefix[0], chunk_prefix[1]);
273 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
std::set< ChunkKey >::iterator eraseChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::createTrackerMapEntryIfNoneExists ( const ChunkKey chunk_key)
private

Definition at line 451 of file ForeignStorageCache.cpp.

References CHECK, eviction_tracker_map_, and is_table_key().

Referenced by cacheTableChunks(), and recoverCacheForTable().

451  {
452  CHECK(is_table_key(table_key));
453  if (eviction_tracker_map_.find(table_key) == eviction_tracker_map_.end()) {
454  eviction_tracker_map_.emplace(table_key, TableEvictionTracker{});
455  }
456 }
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists ( const ChunkKey chunk_key)

Definition at line 62 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, chunks_mutex_, eviction_tracker_map_, get_table_key(), global_file_mgr_, and metadata_mutex_.

62  {
63  write_lock meta_lock(metadata_mutex_);
64  write_lock chunk_lock(chunks_mutex_);
65  if (cached_metadata_.find(chunk_key) != cached_metadata_.end()) {
66  const auto& tracker_it = eviction_tracker_map_.find(get_table_key(chunk_key));
67  if (tracker_it != eviction_tracker_map_.end()) {
68  tracker_it->second.eviction_alg_->removeChunk(chunk_key);
69  }
70  global_file_mgr_->deleteBuffer(chunk_key);
71  cached_chunks_.erase(chunk_key);
72  cached_metadata_.erase(chunk_key);
73  }
74 }
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries ( ) const

Definition at line 370 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, and show_chunk().

370  {
371  auto timer = DEBUG_TIMER(__func__);
372  std::string ret_string = "Cached chunks:\n";
373  for (const auto& chunk_key : cached_chunks_) {
374  ret_string += " " + show_chunk(chunk_key) + "\n";
375  }
376  return ret_string;
377 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries ( ) const

Definition at line 379 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and show_chunk().

379  {
380  auto timer = DEBUG_TIMER(__func__);
381  std::string ret_string = "Cached ChunkMetadata:\n";
382  for (const auto& meta_key : cached_metadata_) {
383  ret_string += " " + show_chunk(meta_key) + "\n";
384  }
385  return ret_string;
386 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpEvictionQueue ( ) const

Definition at line 388 of file ForeignStorageCache.cpp.

References eviction_tracker_map_, and show_chunk().

388  {
389  std::string ret;
390  for (auto& [key, tracker] : eviction_tracker_map_) {
391  auto& [alg, num_pages] = tracker;
392  ret += "queue for table_key: " + show_chunk(key) + "\n" +
393  ((LRUEvictionAlgorithm*)alg.get())->dumpEvictionQueue();
394  }
395 
396  return ret;
397 }
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73

+ Here is the call graph for this function:

std::set<ChunkKey>::iterator foreign_storage::ForeignStorageCache::eraseChunk ( const std::set< ChunkKey >::iterator &  )
private

Referenced by evictThenEraseChunkUnlocked(), insertChunkIntoEvictionAlg(), and setLimit().

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey chunk_key,
TableEvictionTracker tracker 
)
private

Definition at line 346 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, File_Namespace::FileBuffer::freeChunkPages(), global_file_mgr_, and foreign_storage::TableEvictionTracker::num_pages_.

347  {
348  auto timer = DEBUG_TIMER(__func__);
349  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
350  return;
351  }
352  File_Namespace::FileBuffer* file_buffer =
353  static_cast<File_Namespace::FileBuffer*>(global_file_mgr_->getBuffer(chunk_key));
354  tracker.num_pages_ -= file_buffer->freeChunkPages();
355  cached_chunks_.erase(chunk_key);
356 }
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

std::set< ChunkKey >::iterator foreign_storage::ForeignStorageCache::eraseChunkByIterator ( const std::set< ChunkKey >::iterator &  chunk_it)
private

Definition at line 358 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, eviction_tracker_map_, File_Namespace::FileBuffer::freeChunkPages(), get_table_key(), and global_file_mgr_.

Referenced by clear(), and clearForTablePrefix().

359  {
360  auto timer = DEBUG_TIMER(__func__);
361  const ChunkKey table_key = get_table_key(*chunk_it);
362  auto& [eviction_alg, num_pages] = eviction_tracker_map_.at(table_key);
363  eviction_alg->removeChunk(*chunk_it);
364  File_Namespace::FileBuffer* file_buffer =
365  static_cast<File_Namespace::FileBuffer*>(global_file_mgr_->getBuffer(*chunk_it));
366  num_pages -= file_buffer->freeChunkPages();
367  return cached_chunks_.erase(chunk_it);
368 }
std::vector< int > ChunkKey
Definition: types.h:37
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunk ( const ChunkKey chunk_key)

Definition at line 156 of file ForeignStorageCache.cpp.

References chunks_mutex_, and evictThenEraseChunkUnlocked().

156  {
157  write_lock chunk_lock(chunks_mutex_);
158  evictThenEraseChunkUnlocked(chunk_key);
159 }
void evictThenEraseChunkUnlocked(const ChunkKey &)
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunkUnlocked ( const ChunkKey chunk_key)
private

Definition at line 161 of file ForeignStorageCache.cpp.

References eraseChunk(), eviction_tracker_map_, and get_table_key().

Referenced by cacheMetadataVec(), and evictThenEraseChunk().

161  {
162  const ChunkKey table_prefix = get_table_key(chunk_key);
163  const auto& eviction_tracker_it = eviction_tracker_map_.find(table_prefix);
164  if (eviction_tracker_it != eviction_tracker_map_.end()) {
165  eviction_tracker_it->second.eviction_alg_->removeChunk(chunk_key);
166  eraseChunk(chunk_key, eviction_tracker_it->second);
167  }
168 }
std::vector< int > ChunkKey
Definition: types.h:37
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists ( const ChunkKey chunk_key)

Definition at line 113 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, DEBUG_TIMER, eviction_tracker_map_, get_table_key(), and global_file_mgr_.

113  {
114  auto timer = DEBUG_TIMER(__func__);
115  {
116  read_lock lock(chunks_mutex_);
117  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
118  return nullptr;
119  }
120  }
122  const auto& eviction_tracker_it = eviction_tracker_map_.find(get_table_key(chunk_key));
123  if (eviction_tracker_it != eviction_tracker_map_.end()) {
124  eviction_tracker_it->second.eviction_alg_->touchChunk(chunk_key);
125  }
126 
127  return global_file_mgr_->getBuffer(chunk_key);
128 }
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 316 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

317  {
318  read_lock r_lock(chunks_mutex_);
319  std::vector<ChunkKey> ret_vec;
321  [&ret_vec](auto chunk) { ret_vec.push_back(chunk); }, cached_chunks_, chunk_prefix);
322  return ret_vec;
323 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTablePrefix ( const ChunkKey table_prefix) const

Definition at line 419 of file ForeignStorageCache.cpp.

References CHECK, File_Namespace::GlobalFileMgr::getFileMgr(), and getGlobalFileMgr().

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk().

420  {
421  CHECK(table_prefix.size() >= 2);
422  auto fileMgr = dynamic_cast<File_Namespace::FileMgr*>(
423  getGlobalFileMgr()->getFileMgr(table_prefix));
424  CHECK(fileMgr);
425  return fileMgr->getFileMgrBasePath();
426 }
AbstractBufferMgr * getFileMgr(const int db_id, const int tb_id)
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix ( ChunkMetadataVector metadata_vec,
const ChunkKey chunk_prefix 
) const

Definition at line 220 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, global_file_mgr_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

222  {
223  auto timer = DEBUG_TIMER(__func__);
224  read_lock r_lock(metadata_mutex_);
226  [&metadata_vec, this](auto chunk) {
227  std::shared_ptr<ChunkMetadata> buf_metadata = std::make_shared<ChunkMetadata>();
228  global_file_mgr_->getBuffer(chunk)->getEncoder()->getMetadata(buf_metadata);
229  metadata_vec.push_back(std::make_pair(chunk, buf_metadata));
230  },
232  chunk_prefix);
233 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::map< ChunkKey, AbstractBuffer * > foreign_storage::ForeignStorageCache::getChunkBuffersForCaching ( const std::vector< ChunkKey > &  chunk_keys) const

Definition at line 325 of file ForeignStorageCache.cpp.

References cached_chunks_, CHECK, CHECK_EQ, chunks_mutex_, DEBUG_TIMER, and global_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

326  {
327  auto timer = DEBUG_TIMER(__func__);
328  std::map<ChunkKey, AbstractBuffer*> chunk_buffer_map;
329  read_lock lock(chunks_mutex_);
330  for (const auto& chunk_key : chunk_keys) {
331  CHECK(cached_chunks_.find(chunk_key) == cached_chunks_.end());
332  CHECK(global_file_mgr_->isBufferOnDevice(chunk_key));
333  chunk_buffer_map[chunk_key] = global_file_mgr_->getBuffer(chunk_key);
334  CHECK(dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[chunk_key]));
335  CHECK_EQ(chunk_buffer_map[chunk_key]->pageCount(), static_cast<size_t>(0));
336 
337  // Clear all buffer metadata
338  chunk_buffer_map[chunk_key]->resetToEmpty();
339  }
340  return chunk_buffer_map;
341 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the caller graph for this function:

File_Namespace::GlobalFileMgr* foreign_storage::ForeignStorageCache::getGlobalFileMgr ( ) const
inline

Definition at line 111 of file ForeignStorageCache.h.

Referenced by getCacheDirectoryForTablePrefix().

111  {
112  return global_file_mgr_.get();
113  }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_

+ Here is the caller graph for this function:

uint64_t foreign_storage::ForeignStorageCache::getLimit ( ) const
inline

Definition at line 98 of file ForeignStorageCache.h.

98  {
99  return max_pages_per_table_ * global_file_mgr_->getDefaultPageSize();
100  }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const
inline

Definition at line 101 of file ForeignStorageCache.h.

101 { return cached_chunks_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const
inline

Definition at line 102 of file ForeignStorageCache.h.

102 { return cached_metadata_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumChunksAdded ( ) const
inline

Definition at line 103 of file ForeignStorageCache.h.

size_t foreign_storage::ForeignStorageCache::getNumMetadataAdded ( ) const
inline

Definition at line 104 of file ForeignStorageCache.h.

bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 235 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

236  {
237  auto timer = DEBUG_TIMER(__func__);
239  // We don't use iterateOvermatchingPrefix() here because we want to exit early if
240  // possible.
241  ChunkKey upper_prefix(chunk_prefix);
242  upper_prefix.push_back(std::numeric_limits<int>::max());
243  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
244  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;
245  ++meta_it) {
246  return true;
247  }
248  return false;
249 }
std::vector< int > ChunkKey
Definition: types.h:37
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::insertChunkIntoEvictionAlg ( const ChunkKey chunk_key,
const size_t  chunk_size 
)
private

Definition at line 428 of file ForeignStorageCache.cpp.

References cached_chunks_, CHECK, eraseChunk(), eviction_tracker_map_, get_table_key(), global_file_mgr_, and max_pages_per_table_.

Referenced by cacheChunk(), cacheTableChunks(), and recoverCacheForTable().

429  {
430  size_t page_size = global_file_mgr_->getDefaultPageSize();
431  const ChunkKey table_key = get_table_key(chunk_key);
432  auto&& tracker_it = eviction_tracker_map_.find(table_key);
433  CHECK(tracker_it != eviction_tracker_map_.end());
434  auto& tracker = tracker_it->second;
435  // number of pages per chunk rounded up.
436  size_t num_pages_for_chunk = (chunk_size + (page_size - 1)) / page_size;
437  if (num_pages_for_chunk > max_pages_per_table_) {
438  // Can't fit the chunk in the cache, so bail.
439  return false;
440  }
441  while (tracker.num_pages_ + num_pages_for_chunk > max_pages_per_table_) {
442  eraseChunk(tracker.eviction_alg_->evictNextChunk(), tracker);
443  }
444 
445  tracker.eviction_alg_->touchChunk(chunk_key);
446  cached_chunks_.emplace(chunk_key);
447  tracker.num_pages_ += num_pages_for_chunk;
448  return true;
449 }
std::vector< int > ChunkKey
Definition: types.h:37
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached ( const ChunkKey chunk_key) const

Definition at line 130 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

130  {
131  auto timer = DEBUG_TIMER(__func__);
133  return (cached_metadata_.find(chunk_key) != cached_metadata_.end());
134 }
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::recoverCacheForTable ( ChunkMetadataVector meta_vec,
const ChunkKey table_key 
)

Definition at line 136 of file ForeignStorageCache.cpp.

References cached_metadata_, CHECK, chunks_mutex_, createTrackerMapEntryIfNoneExists(), global_file_mgr_, insertChunkIntoEvictionAlg(), and is_table_key().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

137  {
139  CHECK(meta_vec.size() == 0);
140  CHECK(is_table_key(table_key));
141  CHECK(dynamic_cast<File_Namespace::FileMgr*>(global_file_mgr_->getFileMgr(table_key)));
142 
144  global_file_mgr_->getChunkMetadataVecForKeyPrefix(meta_vec, table_key);
145  for (auto& [chunk_key, metadata] : meta_vec) {
146  cached_metadata_.emplace(chunk_key);
147  // If there is no page count then the chunk was metadata only and should not be
148  // cached.
149  if (const auto& buf = global_file_mgr_->getBuffer(chunk_key); buf->pageCount() > 0) {
150  insertChunkIntoEvictionAlg(chunk_key, buf->size());
151  }
152  }
153  return (meta_vec.size() > 0);
154 }
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
bool insertChunkIntoEvictionAlg(const ChunkKey &, const size_t)
void createTrackerMapEntryIfNoneExists(const ChunkKey &chunk_key)
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::setLimit ( uint64_t  limit)

Definition at line 296 of file ForeignStorageCache.cpp.

References chunks_mutex_, DEBUG_TIMER, eraseChunk(), eviction_tracker_map_, omnisci::file_size(), global_file_mgr_, max_cached_bytes_, MAX_FILE_N_PAGES, max_pages_per_table_, and to_string().

Referenced by ForeignStorageCache().

296  {
297  auto timer = DEBUG_TIMER(__func__);
298  write_lock w_lock(chunks_mutex_);
299  size_t file_size = global_file_mgr_->getDefaultPageSize() * MAX_FILE_N_PAGES;
300  if (limit < file_size) {
301  throw CacheTooSmallException("Could not create cache with size " + to_string(limit) +
302  ". Minimum cache size is " + to_string(file_size));
303  }
304  size_t max_num_files = (limit + (file_size - 1)) / file_size;
305  max_pages_per_table_ = max_num_files * MAX_FILE_N_PAGES;
306  for (auto& [table_key, tracker] : eviction_tracker_map_) {
307  auto& [eviction_alg, num_pages] = tracker;
308  while (num_pages > max_pages_per_table_) {
309  eraseChunk(eviction_alg->evictNextChunk(), tracker);
310  }
311  }
312  global_file_mgr_->checkpoint();
313  max_cached_bytes_ = limit;
314 }
std::map< const ChunkKey, TableEvictionTracker > eviction_tracker_map_
std::string to_string(char const *&&v)
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)
#define MAX_FILE_N_PAGES
Definition: File.h:26
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string &  base_path) const
private

Definition at line 399 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

399  {
400  // check if base_path already exists, and if not create one
401  boost::filesystem::path path(base_path);
402  if (boost::filesystem::exists(path)) {
403  if (!boost::filesystem::is_directory(path)) {
404  throw std::runtime_error{
405  "cache path \"" + base_path +
406  "\" is not a directory. Please specify a valid directory "
407  "with --disk_cache_path=<path>, or use the default location."};
408  }
409  } else { // data directory does not exist
410  if (!boost::filesystem::create_directory(path)) {
411  throw std::runtime_error{
412  "could not create directory at cache path \"" + base_path +
413  "\". Please specify a valid directory location "
414  "with --disk_cache_path=<path> or use the default location."};
415  }
416  }
417 }

+ Here is the caller graph for this function:

Member Data Documentation

uint64_t foreign_storage::ForeignStorageCache::max_cached_bytes_
private

Definition at line 150 of file ForeignStorageCache.h.

Referenced by setLimit().

uint64_t foreign_storage::ForeignStorageCache::max_pages_per_table_
private

Definition at line 132 of file ForeignStorageCache.h.

Referenced by insertChunkIntoEvictionAlg(), and setLimit().

size_t foreign_storage::ForeignStorageCache::num_chunks_added_
private

Definition at line 142 of file ForeignStorageCache.h.

Referenced by cacheChunk(), and cacheTableChunks().

size_t foreign_storage::ForeignStorageCache::num_metadata_added_
private

Definition at line 143 of file ForeignStorageCache.h.

Referenced by cacheMetadataVec().


The documentation for this class was generated from the following files: