OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::ForeignStorageCache Class Reference

#include <ForeignStorageCache.h>

Public Member Functions

 ForeignStorageCache (const DiskCacheConfig &config)
 
void cacheTableChunks (const std::vector< ChunkKey > &chunk_keys)
 
void cacheChunk (const ChunkKey &, AbstractBuffer *)
 
File_Namespace::FileBuffergetCachedChunkIfExists (const ChunkKey &)
 
bool isMetadataCached (const ChunkKey &) const
 
void cacheMetadataVec (const ChunkMetadataVector &)
 
void getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const
 
bool hasCachedMetadataForKeyPrefix (const ChunkKey &) const
 
void clearForTablePrefix (const ChunkKey &)
 
void clear ()
 
std::vector< ChunkKeygetCachedChunksForKeyPrefix (const ChunkKey &) const
 
bool recoverCacheForTable (ChunkMetadataVector &, const ChunkKey &)
 
ChunkToBufferMap getChunkBuffersForCaching (const std::vector< ChunkKey > &chunk_keys) const
 
AbstractBuffergetChunkBufferForPrecaching (const ChunkKey &chunk_key, bool is_new_buffer)
 
void deleteBufferIfExists (const ChunkKey &chunk_key)
 
size_t getNumCachedChunks () const
 
size_t getNumCachedMetadata () const
 
size_t getNumChunksAdded () const
 
size_t getNumMetadataAdded () const
 
std::string dumpCachedChunkEntries () const
 
std::string dumpCachedMetadataEntries () const
 
std::string getCacheDirectory () const
 
std::string getCacheDirectoryForTable (int db_id, int tb_id) const
 
void cacheMetadataWithFragIdGreaterOrEqualTo (const ChunkMetadataVector &metadata_vec, const int frag_id)
 
void evictThenEraseChunk (const ChunkKey &)
 
uint64_t getSpaceReservedByTable (int db_id, int tb_id) const
 

Private Member Functions

std::set< ChunkKey >::iterator eraseChunk (const std::set< ChunkKey >::iterator &)
 
void eraseChunk (const ChunkKey &chunk_key)
 
std::set< ChunkKey >::iterator evictChunkByIterator (const std::set< ChunkKey >::iterator &chunk_it)
 
void evictThenEraseChunkUnlocked (const ChunkKey &)
 
void validatePath (const std::string &) const
 

Private Attributes

std::unique_ptr
< File_Namespace::CachingFileMgr
caching_file_mgr_
 
std::set< ChunkKeycached_chunks_
 
std::set< ChunkKeycached_metadata_
 
size_t num_chunks_added_
 
size_t num_metadata_added_
 
mapd_shared_mutex chunks_mutex_
 
mapd_shared_mutex metadata_mutex_
 

Detailed Description

Definition at line 60 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache ( const DiskCacheConfig config)

Definition at line 54 of file ForeignStorageCache.cpp.

References caching_file_mgr_, DiskCacheConfig::num_reader_threads, DiskCacheConfig::page_size, DiskCacheConfig::path, and validatePath().

56  validatePath(config.path);
57  caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(
58  config.path, config.num_reader_threads, config.page_size);
59 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
void validatePath(const std::string &) const

+ Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheChunk ( const ChunkKey chunk_key,
AbstractBuffer buffer 
)

Definition at line 69 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, deleteBufferIfExists(), Data_Namespace::AbstractBuffer::isDirty(), metadata_mutex_, num_chunks_added_, Data_Namespace::AbstractBuffer::setAppended(), and Data_Namespace::AbstractBuffer::size().

69  {
70  // We should only be caching buffers that are in sync with storage.
71  CHECK(!buffer->isDirty());
73  if (buffer->size() == 0) {
74  // If we are writing an empty buffer, just delete it from the cache entirely.
75  deleteBufferIfExists(chunk_key);
76  } else {
77  // Replace the existing chunk with a new version.
78  write_lock meta_lock(metadata_mutex_);
79  write_lock chunk_lock(chunks_mutex_);
80  buffer->setAppended();
81  caching_file_mgr_->putBuffer(chunk_key, buffer);
82  cached_metadata_.emplace(chunk_key);
83  cached_chunks_.emplace(chunk_key);
84  CHECK(!buffer->isDirty());
85  }
86  caching_file_mgr_->checkpoint(chunk_key[CHUNK_KEY_DB_IDX],
87  chunk_key[CHUNK_KEY_TABLE_IDX]);
88 }
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
void deleteBufferIfExists(const ChunkKey &chunk_key)
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataVec ( const ChunkMetadataVector metadata_vec)

Definition at line 170 of file ForeignStorageCache.cpp.

References cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, evictThenEraseChunkUnlocked(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), in_same_table(), is_varlen_data_key(), is_varlen_key(), metadata_mutex_, num_metadata_added_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by cacheMetadataWithFragIdGreaterOrEqualTo(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

170  {
171  auto timer = DEBUG_TIMER(__func__);
172  if (metadata_vec.empty()) {
173  return;
174  }
175  auto first_chunk_key = metadata_vec.begin()->first;
176  write_lock meta_lock(metadata_mutex_);
177  write_lock chunk_lock(chunks_mutex_);
178  for (auto& [chunk_key, metadata] : metadata_vec) {
179  CHECK(in_same_table(chunk_key, first_chunk_key));
180  cached_metadata_.emplace(chunk_key);
181  AbstractBuffer* buf;
182  AbstractBuffer* index_buffer = nullptr;
183  ChunkKey index_chunk_key;
184  if (is_varlen_key(chunk_key)) {
185  // For variable length chunks, metadata is associated with the data chunk.
186  CHECK(is_varlen_data_key(chunk_key));
187  index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
188  chunk_key[CHUNK_KEY_TABLE_IDX],
189  chunk_key[CHUNK_KEY_COLUMN_IDX],
190  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
191  2};
192  }
193  bool chunk_in_cache = false;
194  if (!caching_file_mgr_->isBufferOnDevice(chunk_key)) {
195  buf = caching_file_mgr_->createBuffer(chunk_key);
196 
197  if (!index_chunk_key.empty()) {
198  CHECK(!caching_file_mgr_->isBufferOnDevice(index_chunk_key));
199  index_buffer = caching_file_mgr_->createBuffer(index_chunk_key);
200  CHECK(index_buffer);
201  }
202  } else {
203  buf = caching_file_mgr_->getBuffer(chunk_key);
204 
205  if (!index_chunk_key.empty()) {
206  CHECK(caching_file_mgr_->isBufferOnDevice(index_chunk_key));
207  index_buffer = caching_file_mgr_->getBuffer(index_chunk_key);
208  CHECK(index_buffer);
209  }
210 
211  // We should have already cleared the data unless we are appending
212  // If the buffer metadata has changed, we need to remove this chunk
213  if (buf->getEncoder() != nullptr) {
214  const std::shared_ptr<ChunkMetadata> buf_metadata =
215  std::make_shared<ChunkMetadata>();
216  buf->getEncoder()->getMetadata(buf_metadata);
217  chunk_in_cache = *metadata.get() == *buf_metadata;
218  }
219  }
220 
221  if (!chunk_in_cache) {
222  set_metadata_for_buffer(buf, metadata.get());
223  evictThenEraseChunkUnlocked(chunk_key);
224 
225  if (!index_chunk_key.empty()) {
226  CHECK(index_buffer);
227  index_buffer->setUpdated();
228  evictThenEraseChunkUnlocked(index_chunk_key);
229  }
230  }
232  }
233  caching_file_mgr_->checkpoint(first_chunk_key[CHUNK_KEY_DB_IDX],
234  first_chunk_key[CHUNK_KEY_TABLE_IDX]);
235 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:70
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:227
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
An AbstractBuffer is a unit of data management for a data manager.
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
void evictThenEraseChunkUnlocked(const ChunkKey &)
void set_metadata_for_buffer(AbstractBuffer *buffer, ChunkMetadata *meta)
#define CHECK(condition)
Definition: Logger.h:203
#define DEBUG_TIMER(name)
Definition: Logger.h:319
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool in_same_table(const ChunkKey &left_key, const ChunkKey &right_key)
Definition: types.h:78
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:66

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataWithFragIdGreaterOrEqualTo ( const ChunkMetadataVector metadata_vec,
const int  frag_id 
)

Definition at line 398 of file ForeignStorageCache.cpp.

References cacheMetadataVec(), and CHUNK_KEY_FRAGMENT_IDX.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache().

400  {
401  // Only re-cache last fragment and above
402  ChunkMetadataVector new_metadata_vec;
403  for (const auto& chunk_metadata : metadata_vec) {
404  if (chunk_metadata.first[CHUNK_KEY_FRAGMENT_IDX] >= frag_id) {
405  new_metadata_vec.push_back(chunk_metadata);
406  }
407  }
408  cacheMetadataVec(new_metadata_vec);
409 }
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void cacheMetadataVec(const ChunkMetadataVector &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheTableChunks ( const std::vector< ChunkKey > &  chunk_keys)

Caches the chunks for the given chunk keys. Chunk buffers for chunks to be cached are expected to have already been populated before calling this method. This method also expects all provided chunk keys to be for the same table.

Parameters
chunk_keys- keys of chunks to be cached

Definition at line 90 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, CHECK, CHECK_EQ, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, and num_chunks_added_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

90  {
91  auto timer = DEBUG_TIMER(__func__);
93  CHECK(!chunk_keys.empty());
94 
95  auto db_id = chunk_keys[0][CHUNK_KEY_DB_IDX];
96  auto table_id = chunk_keys[0][CHUNK_KEY_TABLE_IDX];
97  const ChunkKey table_key{db_id, table_id};
98 
99  for (const auto& chunk_key : chunk_keys) {
100  CHECK_EQ(db_id, chunk_key[CHUNK_KEY_DB_IDX]);
101  CHECK_EQ(table_id, chunk_key[CHUNK_KEY_TABLE_IDX]);
102  CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
104  cached_chunks_.emplace(chunk_key);
105  }
106  caching_file_mgr_->checkpoint(db_id, table_id);
107 }
#define CHECK_EQ(x, y)
Definition: Logger.h:211
std::vector< int > ChunkKey
Definition: types.h:37
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:203
#define DEBUG_TIMER(name)
Definition: Logger.h:319
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear ( )

Definition at line 292 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), and metadata_mutex_.

292  {
293  auto timer = DEBUG_TIMER(__func__);
294  std::set<ChunkKey> table_keys;
295  {
296  write_lock w_lock(chunks_mutex_);
297  for (auto chunk_it = cached_chunks_.begin(); chunk_it != cached_chunks_.end();) {
298  chunk_it = evictChunkByIterator(chunk_it);
299  }
300  }
301  {
302  write_lock w_lock(metadata_mutex_);
303  for (auto meta_it = cached_metadata_.begin(); meta_it != cached_metadata_.end();) {
304  table_keys.emplace(ChunkKey{(*meta_it)[0], (*meta_it)[1]});
305  meta_it = cached_metadata_.erase(meta_it);
306  }
307  }
308  // FileMgrs do not clean up after themselves nicely, so we need to close all their disk
309  // resources and then re-create the CachingFileMgr to reset it.
310  caching_file_mgr_->closeRemovePhysical();
311  boost::filesystem::create_directory(caching_file_mgr_->getFileMgrBasePath());
312  caching_file_mgr_ = std::make_unique<File_Namespace::CachingFileMgr>(
313  caching_file_mgr_->getFileMgrBasePath(), caching_file_mgr_->getNumReaderThreads());
314 }
std::vector< int > ChunkKey
Definition: types.h:37
std::set< ChunkKey >::iterator evictChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:319
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::clearForTablePrefix ( const ChunkKey chunk_prefix)

Definition at line 267 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), is_table_key(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::clearTable().

267  {
268  CHECK(is_table_key(chunk_prefix));
269  auto timer = DEBUG_TIMER(__func__);
270  ChunkKey upper_prefix(chunk_prefix);
271  upper_prefix.push_back(std::numeric_limits<int>::max());
272  {
273  write_lock w_lock(chunks_mutex_);
274  // Delete chunks for prefix
275  auto end_it = cached_chunks_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
276  for (auto chunk_it = cached_chunks_.lower_bound(chunk_prefix); chunk_it != end_it;) {
277  chunk_it = evictChunkByIterator(chunk_it);
278  }
279  }
280  {
281  write_lock w_lock(metadata_mutex_);
282  // Delete metadata for prefix
283  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
284  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;) {
285  meta_it = cached_metadata_.erase(meta_it);
286  }
287  }
288  caching_file_mgr_->clearForTable(chunk_prefix[CHUNK_KEY_DB_IDX],
289  chunk_prefix[CHUNK_KEY_TABLE_IDX]);
290 }
std::vector< int > ChunkKey
Definition: types.h:37
std::set< ChunkKey >::iterator evictChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:203
#define DEBUG_TIMER(name)
Definition: Logger.h:319
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists ( const ChunkKey chunk_key)

Definition at line 61 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, chunks_mutex_, and metadata_mutex_.

Referenced by cacheChunk().

61  {
62  write_lock meta_lock(metadata_mutex_);
63  write_lock chunk_lock(chunks_mutex_);
64  caching_file_mgr_->deleteBufferIfExists(chunk_key);
65  cached_chunks_.erase(chunk_key);
66  cached_metadata_.erase(chunk_key);
67 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries ( ) const

Definition at line 362 of file ForeignStorageCache.cpp.

References cached_chunks_, and show_chunk().

362  {
363  std::string ret_string = "Cached chunks:\n";
364  for (const auto& chunk_key : cached_chunks_) {
365  ret_string += " " + show_chunk(chunk_key) + "\n";
366  }
367  return ret_string;
368 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:85

+ Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries ( ) const

Definition at line 370 of file ForeignStorageCache.cpp.

References cached_metadata_, and show_chunk().

370  {
371  std::string ret_string = "Cached ChunkMetadata:\n";
372  for (const auto& meta_key : cached_metadata_) {
373  ret_string += " " + show_chunk(meta_key) + "\n";
374  }
375  return ret_string;
376 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:85

+ Here is the call graph for this function:

std::set<ChunkKey>::iterator foreign_storage::ForeignStorageCache::eraseChunk ( const std::set< ChunkKey >::iterator &  )
private

Referenced by evictThenEraseChunkUnlocked().

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey chunk_key)
private

Definition at line 344 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and File_Namespace::FileBuffer::freeChunkPages().

344  {
345  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
346  return;
347  }
348  File_Namespace::FileBuffer* file_buffer =
349  static_cast<File_Namespace::FileBuffer*>(caching_file_mgr_->getBuffer(chunk_key));
350  file_buffer->freeChunkPages();
351  cached_chunks_.erase(chunk_key);
352 }
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:58
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

+ Here is the call graph for this function:

std::set< ChunkKey >::iterator foreign_storage::ForeignStorageCache::evictChunkByIterator ( const std::set< ChunkKey >::iterator &  chunk_it)
private

Definition at line 354 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and File_Namespace::FileBuffer::freeChunkPages().

Referenced by clear(), and clearForTablePrefix().

355  {
356  File_Namespace::FileBuffer* file_buffer =
357  static_cast<File_Namespace::FileBuffer*>(caching_file_mgr_->getBuffer(*chunk_it));
358  file_buffer->freeChunkPages();
359  return cached_chunks_.erase(chunk_it);
360 }
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:58
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunk ( const ChunkKey chunk_key)

Definition at line 160 of file ForeignStorageCache.cpp.

References chunks_mutex_, and evictThenEraseChunkUnlocked().

160  {
161  write_lock chunk_lock(chunks_mutex_);
162  evictThenEraseChunkUnlocked(chunk_key);
163 }
void evictThenEraseChunkUnlocked(const ChunkKey &)
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunkUnlocked ( const ChunkKey chunk_key)
private

Definition at line 165 of file ForeignStorageCache.cpp.

References eraseChunk(), and get_table_key().

Referenced by cacheMetadataVec(), and evictThenEraseChunk().

165  {
166  const ChunkKey table_prefix = get_table_key(chunk_key);
167  eraseChunk(chunk_key);
168 }
std::vector< int > ChunkKey
Definition: types.h:37
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

File_Namespace::FileBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists ( const ChunkKey chunk_key)

Definition at line 109 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, and chunks_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

110  {
111  {
112  read_lock lock(chunks_mutex_);
113  // We do this instead of calling getBuffer so that we don't create a fileMgr if the
114  // chunk doesn't exist.
115  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
116  return nullptr;
117  }
118  }
119  return caching_file_mgr_->getBuffer(chunk_key);
120 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the caller graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 316 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

317  {
318  read_lock r_lock(chunks_mutex_);
319  std::vector<ChunkKey> ret_vec;
321  [&ret_vec](auto chunk) { ret_vec.push_back(chunk); }, cached_chunks_, chunk_prefix);
322  return ret_vec;
323 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectory ( ) const
inline

Definition at line 102 of file ForeignStorageCache.h.

102  {
103  return caching_file_mgr_->getFileMgrBasePath();
104  }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTable ( int  db_id,
int  tb_id 
) const
inline

Definition at line 106 of file ForeignStorageCache.h.

Referenced by foreign_storage::CachingForeignStorageMgr::clearTable(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk().

106  {
107  return caching_file_mgr_->getOrAddTableDir(db_id, tb_id);
108  }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix ( ChunkMetadataVector metadata_vec,
const ChunkKey chunk_prefix 
) const

Definition at line 237 of file ForeignStorageCache.cpp.

References cached_metadata_, caching_file_mgr_, DEBUG_TIMER, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

239  {
240  auto timer = DEBUG_TIMER(__func__);
241  read_lock r_lock(metadata_mutex_);
243  [&metadata_vec, this](auto chunk) {
244  std::shared_ptr<ChunkMetadata> buf_metadata = std::make_shared<ChunkMetadata>();
245  caching_file_mgr_->getBuffer(chunk)->getEncoder()->getMetadata(buf_metadata);
246  metadata_vec.push_back(std::make_pair(chunk, buf_metadata));
247  },
249  chunk_prefix);
250 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:319

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getChunkBufferForPrecaching ( const ChunkKey chunk_key,
bool  is_new_buffer 
)

Definition at line 411 of file ForeignStorageCache.cpp.

References caching_file_mgr_, and CHECK.

413  {
414  if (!is_new_buffer) {
415  CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
416  return caching_file_mgr_->getBuffer(chunk_key);
417  } else {
418  CHECK(!caching_file_mgr_->isBufferOnDevice(chunk_key));
419  return caching_file_mgr_->createBuffer(chunk_key);
420  }
421 }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:203
ChunkToBufferMap foreign_storage::ForeignStorageCache::getChunkBuffersForCaching ( const std::vector< ChunkKey > &  chunk_keys) const

Definition at line 325 of file ForeignStorageCache.cpp.

References cached_chunks_, caching_file_mgr_, CHECK, CHECK_EQ, and chunks_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

326  {
327  ChunkToBufferMap chunk_buffer_map;
328  read_lock lock(chunks_mutex_);
329  for (const auto& chunk_key : chunk_keys) {
330  CHECK(cached_chunks_.find(chunk_key) == cached_chunks_.end());
331  CHECK(caching_file_mgr_->isBufferOnDevice(chunk_key));
332  chunk_buffer_map[chunk_key] = caching_file_mgr_->getBuffer(chunk_key);
333  CHECK(dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[chunk_key]));
334  CHECK_EQ(chunk_buffer_map[chunk_key]->pageCount(), static_cast<size_t>(0));
335 
336  // Clear all buffer metadata
337  chunk_buffer_map[chunk_key]->resetToEmpty();
338  }
339  return chunk_buffer_map;
340 }
#define CHECK_EQ(x, y)
Definition: Logger.h:211
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:203

+ Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const
inline

Definition at line 93 of file ForeignStorageCache.h.

93 { return cached_chunks_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const
inline

Definition at line 94 of file ForeignStorageCache.h.

94 { return cached_metadata_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumChunksAdded ( ) const
inline

Definition at line 95 of file ForeignStorageCache.h.

size_t foreign_storage::ForeignStorageCache::getNumMetadataAdded ( ) const
inline

Definition at line 96 of file ForeignStorageCache.h.

uint64_t foreign_storage::ForeignStorageCache::getSpaceReservedByTable ( int  db_id,
int  tb_id 
) const
inline

Definition at line 114 of file ForeignStorageCache.h.

114  {
115  return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
116  }
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 252 of file ForeignStorageCache.cpp.

References cached_metadata_, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

253  {
255  // We don't use iterateOvermatchingPrefix() here because we want to exit early if
256  // possible.
257  ChunkKey upper_prefix(chunk_prefix);
258  upper_prefix.push_back(std::numeric_limits<int>::max());
259  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
260  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;
261  ++meta_it) {
262  return true;
263  }
264  return false;
265 }
std::vector< int > ChunkKey
Definition: types.h:37
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached ( const ChunkKey chunk_key) const

Definition at line 122 of file ForeignStorageCache.cpp.

References cached_metadata_, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

122  {
124  return (cached_metadata_.find(chunk_key) != cached_metadata_.end());
125 }
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::recoverCacheForTable ( ChunkMetadataVector meta_vec,
const ChunkKey table_key 
)

Definition at line 127 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, caching_file_mgr_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, is_table_key(), is_varlen_data_key(), and is_varlen_key().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

128  {
130  CHECK(meta_vec.size() == 0);
131  CHECK(is_table_key(table_key));
132 
133  caching_file_mgr_->getChunkMetadataVecForKeyPrefix(meta_vec, table_key);
134  for (auto& [chunk_key, metadata] : meta_vec) {
135  cached_metadata_.emplace(chunk_key);
136  // If there is no page count then the chunk was metadata only and should not be
137  // cached.
138  if (const auto& buf = caching_file_mgr_->getBuffer(chunk_key); buf->pageCount() > 0) {
139  cached_chunks_.emplace(chunk_key);
140  }
141 
142  if (is_varlen_key(chunk_key)) {
143  // Metadata is only available for the data chunk, but look for the index as well
144  CHECK(is_varlen_data_key(chunk_key));
145  ChunkKey index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
146  chunk_key[CHUNK_KEY_TABLE_IDX],
147  chunk_key[CHUNK_KEY_COLUMN_IDX],
148  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
149  2};
150 
151  if (const auto& buf = caching_file_mgr_->getBuffer(index_chunk_key);
152  buf->pageCount() > 0) {
153  cached_chunks_.emplace(index_chunk_key);
154  }
155  }
156  }
157  return (meta_vec.size() > 0);
158 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:70
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:66

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string &  base_path) const
private

Definition at line 378 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

378  {
379  // check if base_path already exists, and if not create one
380  boost::filesystem::path path(base_path);
381  if (boost::filesystem::exists(path)) {
382  if (!boost::filesystem::is_directory(path)) {
383  throw std::runtime_error{
384  "cache path \"" + base_path +
385  "\" is not a directory. Please specify a valid directory "
386  "with --disk_cache_path=<path>, or use the default location."};
387  }
388  } else { // data directory does not exist
389  if (!boost::filesystem::create_directory(path)) {
390  throw std::runtime_error{
391  "could not create directory at cache path \"" + base_path +
392  "\". Please specify a valid directory location "
393  "with --disk_cache_path=<path> or use the default location."};
394  }
395  }
396 }

+ Here is the caller graph for this function:

Member Data Documentation

size_t foreign_storage::ForeignStorageCache::num_chunks_added_
private

Definition at line 135 of file ForeignStorageCache.h.

Referenced by cacheChunk(), and cacheTableChunks().

size_t foreign_storage::ForeignStorageCache::num_metadata_added_
private

Definition at line 136 of file ForeignStorageCache.h.

Referenced by cacheMetadataVec().


The documentation for this class was generated from the following files: