OmniSciDB  bf83d84833
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::ForeignStorageCache Class Reference

#include <ForeignStorageCache.h>

Public Member Functions

 ForeignStorageCache (const DiskCacheConfig &config)
 
void cacheTableChunks (const std::vector< ChunkKey > &chunk_keys)
 
void cacheChunk (const ChunkKey &, AbstractBuffer *)
 
AbstractBuffergetCachedChunkIfExists (const ChunkKey &)
 
bool isMetadataCached (const ChunkKey &) const
 
void cacheMetadataVec (const ChunkMetadataVector &)
 
void getCachedMetadataVecForKeyPrefix (ChunkMetadataVector &, const ChunkKey &) const
 
bool hasCachedMetadataForKeyPrefix (const ChunkKey &) const
 
void clearForTablePrefix (const ChunkKey &)
 
void clear ()
 
std::vector< ChunkKeygetCachedChunksForKeyPrefix (const ChunkKey &) const
 
bool recoverCacheForTable (ChunkMetadataVector &, const ChunkKey &)
 
std::map< ChunkKey,
AbstractBuffer * > 
getChunkBuffersForCaching (const std::vector< ChunkKey > &chunk_keys) const
 
AbstractBuffergetChunkBufferForPrecaching (const ChunkKey &chunk_key, bool is_new_buffer)
 
void deleteBufferIfExists (const ChunkKey &chunk_key)
 
size_t getNumCachedChunks () const
 
size_t getNumCachedMetadata () const
 
size_t getNumChunksAdded () const
 
size_t getNumMetadataAdded () const
 
std::string dumpCachedChunkEntries () const
 
std::string dumpCachedMetadataEntries () const
 
File_Namespace::GlobalFileMgrgetGlobalFileMgr () const
 
std::string getCacheDirectoryForTablePrefix (const ChunkKey &) const
 
void cacheMetadataWithFragIdGreaterOrEqualTo (const ChunkMetadataVector &metadata_vec, const int frag_id)
 
void evictThenEraseChunk (const ChunkKey &)
 

Private Member Functions

std::set< ChunkKey >::iterator eraseChunk (const std::set< ChunkKey >::iterator &)
 
void eraseChunk (const ChunkKey &chunk_key)
 
std::set< ChunkKey >::iterator evictChunkByIterator (const std::set< ChunkKey >::iterator &chunk_it)
 
void evictThenEraseChunkUnlocked (const ChunkKey &)
 
void validatePath (const std::string &) const
 

Private Attributes

std::unique_ptr
< File_Namespace::GlobalFileMgr
global_file_mgr_
 
std::set< ChunkKeycached_chunks_
 
std::set< ChunkKeycached_metadata_
 
size_t num_chunks_added_
 
size_t num_metadata_added_
 
mapd_shared_mutex chunks_mutex_
 
mapd_shared_mutex metadata_mutex_
 

Detailed Description

Definition at line 58 of file ForeignStorageCache.h.

Constructor & Destructor Documentation

foreign_storage::ForeignStorageCache::ForeignStorageCache ( const DiskCacheConfig config)

Definition at line 54 of file ForeignStorageCache.cpp.

References global_file_mgr_, DiskCacheConfig::num_reader_threads, DiskCacheConfig::path, and validatePath().

56  validatePath(config.path);
57  global_file_mgr_ = std::make_unique<File_Namespace::GlobalFileMgr>(
58  0, config.path, config.num_reader_threads);
59 }
void validatePath(const std::string &) const
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_

+ Here is the call graph for this function:

Member Function Documentation

void foreign_storage::ForeignStorageCache::cacheChunk ( const ChunkKey chunk_key,
AbstractBuffer buffer 
)

Definition at line 71 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, CHECK, chunks_mutex_, DEBUG_TIMER, global_file_mgr_, Data_Namespace::AbstractBuffer::isDirty(), metadata_mutex_, num_chunks_added_, and Data_Namespace::AbstractBuffer::setUpdated().

71  {
72  auto timer = DEBUG_TIMER(__func__);
73  write_lock meta_lock(metadata_mutex_);
74  write_lock chunk_lock(chunks_mutex_);
75  // We should only be caching buffers that are in sync with storage.
76  CHECK(!buffer->isDirty());
77  buffer->setUpdated();
79  global_file_mgr_->putBuffer(chunk_key, buffer);
80  global_file_mgr_->checkpoint();
81  cached_metadata_.emplace(chunk_key);
82  cached_chunks_.emplace(chunk_key);
83  CHECK(!buffer->isDirty());
84 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataVec ( const ChunkMetadataVector metadata_vec)

Definition at line 168 of file ForeignStorageCache.cpp.

References cached_metadata_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, evictThenEraseChunkUnlocked(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), global_file_mgr_, is_varlen_data_key(), is_varlen_key(), metadata_mutex_, num_metadata_added_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Data_Namespace::AbstractBuffer::setUpdated().

Referenced by cacheMetadataWithFragIdGreaterOrEqualTo(), and foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache().

168  {
169  auto timer = DEBUG_TIMER(__func__);
170  write_lock meta_lock(metadata_mutex_);
171  write_lock chunk_lock(chunks_mutex_);
172  for (auto& [chunk_key, metadata] : metadata_vec) {
173  cached_metadata_.emplace(chunk_key);
174  AbstractBuffer* buf;
175  AbstractBuffer* index_buffer = nullptr;
176  ChunkKey index_chunk_key;
177  if (is_varlen_key(chunk_key)) {
178  // For variable length chunks, metadata is associated with the data chunk.
179  CHECK(is_varlen_data_key(chunk_key));
180  index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
181  chunk_key[CHUNK_KEY_TABLE_IDX],
182  chunk_key[CHUNK_KEY_COLUMN_IDX],
183  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
184  2};
185  }
186  bool chunk_in_cache = false;
187  if (!global_file_mgr_->isBufferOnDevice(chunk_key)) {
188  buf = global_file_mgr_->createBuffer(chunk_key);
189 
190  if (!index_chunk_key.empty()) {
191  CHECK(!global_file_mgr_->isBufferOnDevice(index_chunk_key));
192  index_buffer = global_file_mgr_->createBuffer(index_chunk_key);
193  CHECK(index_buffer);
194  }
195  } else {
196  buf = global_file_mgr_->getBuffer(chunk_key);
197 
198  if (!index_chunk_key.empty()) {
199  CHECK(global_file_mgr_->isBufferOnDevice(index_chunk_key));
200  index_buffer = global_file_mgr_->getBuffer(index_chunk_key);
201  CHECK(index_buffer);
202  }
203 
204  // We should have already cleared the data unless we are appending
205  // If the buffer metadata has changed, we need to remove this chunk
206  if (buf->getEncoder() != nullptr) {
207  const std::shared_ptr<ChunkMetadata> buf_metadata =
208  std::make_shared<ChunkMetadata>();
209  buf->getEncoder()->getMetadata(buf_metadata);
210  chunk_in_cache = *metadata.get() == *buf_metadata;
211  }
212  }
213 
214  if (!chunk_in_cache) {
215  set_metadata_for_buffer(buf, metadata.get());
216  evictThenEraseChunkUnlocked(chunk_key);
217 
218  if (!index_chunk_key.empty()) {
219  CHECK(index_buffer);
220  index_buffer->setUpdated();
221  evictThenEraseChunkUnlocked(index_chunk_key);
222  }
223  }
225  }
226  global_file_mgr_->checkpoint();
227 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:65
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:227
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
An AbstractBuffer is a unit of data management for a data manager.
void evictThenEraseChunkUnlocked(const ChunkKey &)
void set_metadata_for_buffer(AbstractBuffer *buffer, ChunkMetadata *meta)
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:61

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheMetadataWithFragIdGreaterOrEqualTo ( const ChunkMetadataVector metadata_vec,
const int  frag_id 
)

Definition at line 401 of file ForeignStorageCache.cpp.

References cacheMetadataVec(), and CHUNK_KEY_FRAGMENT_IDX.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache().

403  {
404  // Only re-cache last fragment and above
405  ChunkMetadataVector new_metadata_vec;
406  for (const auto& chunk_metadata : metadata_vec) {
407  if (chunk_metadata.first[CHUNK_KEY_FRAGMENT_IDX] >= frag_id) {
408  new_metadata_vec.push_back(chunk_metadata);
409  }
410  }
411  cacheMetadataVec(new_metadata_vec);
412 }
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void cacheMetadataVec(const ChunkMetadataVector &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::cacheTableChunks ( const std::vector< ChunkKey > &  chunk_keys)

Caches the chunks for the given chunk keys. Chunk buffers for chunks to be cached are expected to have already been populated before calling this method. This method also expects all provided chunk keys to be for the same table.

Parameters
chunk_keys- keys of chunks to be cached

Definition at line 86 of file ForeignStorageCache.cpp.

References cached_chunks_, CHECK, CHECK_EQ, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, DEBUG_TIMER, global_file_mgr_, and num_chunks_added_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

86  {
87  auto timer = DEBUG_TIMER(__func__);
89  CHECK(!chunk_keys.empty());
90 
91  auto db_id = chunk_keys[0][CHUNK_KEY_DB_IDX];
92  auto table_id = chunk_keys[0][CHUNK_KEY_TABLE_IDX];
93  const ChunkKey table_key{db_id, table_id};
94 
95  for (const auto& chunk_key : chunk_keys) {
96  CHECK_EQ(db_id, chunk_key[CHUNK_KEY_DB_IDX]);
97  CHECK_EQ(table_id, chunk_key[CHUNK_KEY_TABLE_IDX]);
98  CHECK(global_file_mgr_->isBufferOnDevice(chunk_key));
100  cached_chunks_.emplace(chunk_key);
101  }
102  global_file_mgr_->checkpoint(db_id, table_id);
103 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::vector< int > ChunkKey
Definition: types.h:37
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::clear ( )

Definition at line 284 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), global_file_mgr_, and metadata_mutex_.

284  {
285  auto timer = DEBUG_TIMER(__func__);
286  std::set<ChunkKey> table_keys;
287  {
288  write_lock w_lock(chunks_mutex_);
289  for (auto chunk_it = cached_chunks_.begin(); chunk_it != cached_chunks_.end();) {
290  chunk_it = evictChunkByIterator(chunk_it);
291  }
292  }
293  {
294  write_lock w_lock(metadata_mutex_);
295  for (auto meta_it = cached_metadata_.begin(); meta_it != cached_metadata_.end();) {
296  table_keys.emplace(ChunkKey{(*meta_it)[0], (*meta_it)[1]});
297  meta_it = cached_metadata_.erase(meta_it);
298  }
299  }
300  for (const auto& table_key : table_keys) {
301  global_file_mgr_->removeTableRelatedDS(table_key[0], table_key[1]);
302  }
303 }
std::vector< int > ChunkKey
Definition: types.h:37
std::set< ChunkKey >::iterator evictChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::clearForTablePrefix ( const ChunkKey chunk_prefix)

Definition at line 260 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, CHECK, chunks_mutex_, DEBUG_TIMER, evictChunkByIterator(), global_file_mgr_, is_table_key(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache(), and foreign_storage::CachingForeignStorageMgr::refreshTable().

260  {
261  CHECK(is_table_key(chunk_prefix));
262  auto timer = DEBUG_TIMER(__func__);
263  ChunkKey upper_prefix(chunk_prefix);
264  upper_prefix.push_back(std::numeric_limits<int>::max());
265  {
266  write_lock w_lock(chunks_mutex_);
267  // Delete chunks for prefix
268  auto end_it = cached_chunks_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
269  for (auto chunk_it = cached_chunks_.lower_bound(chunk_prefix); chunk_it != end_it;) {
270  chunk_it = evictChunkByIterator(chunk_it);
271  }
272  }
273  {
274  write_lock w_lock(metadata_mutex_);
275  // Delete metadata for prefix
276  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
277  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;) {
278  meta_it = cached_metadata_.erase(meta_it);
279  }
280  }
281  global_file_mgr_->removeTableRelatedDS(chunk_prefix[0], chunk_prefix[1]);
282 }
std::vector< int > ChunkKey
Definition: types.h:37
std::set< ChunkKey >::iterator evictChunkByIterator(const std::set< ChunkKey >::iterator &chunk_it)
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::deleteBufferIfExists ( const ChunkKey chunk_key)

Definition at line 61 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, chunks_mutex_, global_file_mgr_, and metadata_mutex_.

61  {
62  write_lock meta_lock(metadata_mutex_);
63  write_lock chunk_lock(chunks_mutex_);
64  if (cached_metadata_.find(chunk_key) != cached_metadata_.end()) {
65  global_file_mgr_->deleteBuffer(chunk_key);
66  cached_chunks_.erase(chunk_key);
67  cached_metadata_.erase(chunk_key);
68  }
69 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_unique_lock< mapd_shared_mutex > write_lock
std::string foreign_storage::ForeignStorageCache::dumpCachedChunkEntries ( ) const

Definition at line 354 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, and show_chunk().

354  {
355  auto timer = DEBUG_TIMER(__func__);
356  std::string ret_string = "Cached chunks:\n";
357  for (const auto& chunk_key : cached_chunks_) {
358  ret_string += " " + show_chunk(chunk_key) + "\n";
359  }
360  return ret_string;
361 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

std::string foreign_storage::ForeignStorageCache::dumpCachedMetadataEntries ( ) const

Definition at line 363 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and show_chunk().

363  {
364  auto timer = DEBUG_TIMER(__func__);
365  std::string ret_string = "Cached ChunkMetadata:\n";
366  for (const auto& meta_key : cached_metadata_) {
367  ret_string += " " + show_chunk(meta_key) + "\n";
368  }
369  return ret_string;
370 }
std::string show_chunk(const ChunkKey &key)
Definition: types.h:73
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

std::set<ChunkKey>::iterator foreign_storage::ForeignStorageCache::eraseChunk ( const std::set< ChunkKey >::iterator &  )
private

Referenced by evictThenEraseChunkUnlocked().

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::eraseChunk ( const ChunkKey chunk_key)
private

Definition at line 334 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, File_Namespace::FileBuffer::freeChunkPages(), and global_file_mgr_.

334  {
335  auto timer = DEBUG_TIMER(__func__);
336  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
337  return;
338  }
339  File_Namespace::FileBuffer* file_buffer =
340  static_cast<File_Namespace::FileBuffer*>(global_file_mgr_->getBuffer(chunk_key));
341  file_buffer->freeChunkPages();
342  cached_chunks_.erase(chunk_key);
343 }
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:56
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

std::set< ChunkKey >::iterator foreign_storage::ForeignStorageCache::evictChunkByIterator ( const std::set< ChunkKey >::iterator &  chunk_it)
private

Definition at line 345 of file ForeignStorageCache.cpp.

References cached_chunks_, DEBUG_TIMER, File_Namespace::FileBuffer::freeChunkPages(), and global_file_mgr_.

Referenced by clear(), and clearForTablePrefix().

346  {
347  auto timer = DEBUG_TIMER(__func__);
348  File_Namespace::FileBuffer* file_buffer =
349  static_cast<File_Namespace::FileBuffer*>(global_file_mgr_->getBuffer(*chunk_it));
350  file_buffer->freeChunkPages();
351  return cached_chunks_.erase(chunk_it);
352 }
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:56
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunk ( const ChunkKey chunk_key)

Definition at line 158 of file ForeignStorageCache.cpp.

References chunks_mutex_, and evictThenEraseChunkUnlocked().

158  {
159  write_lock chunk_lock(chunks_mutex_);
160  evictThenEraseChunkUnlocked(chunk_key);
161 }
void evictThenEraseChunkUnlocked(const ChunkKey &)
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void foreign_storage::ForeignStorageCache::evictThenEraseChunkUnlocked ( const ChunkKey chunk_key)
private

Definition at line 163 of file ForeignStorageCache.cpp.

References eraseChunk(), and get_table_key().

Referenced by cacheMetadataVec(), and evictThenEraseChunk().

163  {
164  const ChunkKey table_prefix = get_table_key(chunk_key);
165  eraseChunk(chunk_key);
166 }
std::vector< int > ChunkKey
Definition: types.h:37
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
std::set< ChunkKey >::iterator eraseChunk(const std::set< ChunkKey >::iterator &)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getCachedChunkIfExists ( const ChunkKey chunk_key)

Definition at line 105 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, DEBUG_TIMER, and global_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

105  {
106  auto timer = DEBUG_TIMER(__func__);
107  {
108  read_lock lock(chunks_mutex_);
109  // We do this instead of calling getBuffer so that we don't create a fileMgr if the
110  // chunk doesn't exist.
111  if (cached_chunks_.find(chunk_key) == cached_chunks_.end()) {
112  return nullptr;
113  }
114  }
115  return global_file_mgr_->getBuffer(chunk_key);
116 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the caller graph for this function:

std::vector< ChunkKey > foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 305 of file ForeignStorageCache.cpp.

References cached_chunks_, chunks_mutex_, and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

306  {
307  read_lock r_lock(chunks_mutex_);
308  std::vector<ChunkKey> ret_vec;
310  [&ret_vec](auto chunk) { ret_vec.push_back(chunk); }, cached_chunks_, chunk_prefix);
311  return ret_vec;
312 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string foreign_storage::ForeignStorageCache::getCacheDirectoryForTablePrefix ( const ChunkKey table_prefix) const

Definition at line 392 of file ForeignStorageCache.cpp.

References CHECK, File_Namespace::GlobalFileMgr::getFileMgr(), and getGlobalFileMgr().

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk().

393  {
394  CHECK(table_prefix.size() >= 2);
395  auto fileMgr = dynamic_cast<File_Namespace::FileMgr*>(
396  getGlobalFileMgr()->getFileMgr(table_prefix));
397  CHECK(fileMgr);
398  return fileMgr->getFileMgrBasePath();
399 }
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
AbstractBufferMgr * getFileMgr(const int32_t db_id, const int32_t tb_id)
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix ( ChunkMetadataVector metadata_vec,
const ChunkKey chunk_prefix 
) const

Definition at line 229 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, global_file_mgr_, foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::iterate_over_matching_prefix(), and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), and foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId().

231  {
232  auto timer = DEBUG_TIMER(__func__);
233  read_lock r_lock(metadata_mutex_);
235  [&metadata_vec, this](auto chunk) {
236  std::shared_ptr<ChunkMetadata> buf_metadata = std::make_shared<ChunkMetadata>();
237  global_file_mgr_->getBuffer(chunk)->getEncoder()->getMetadata(buf_metadata);
238  metadata_vec.push_back(std::make_pair(chunk, buf_metadata));
239  },
241  chunk_prefix);
242 }
void iterate_over_matching_prefix(Func func, T &chunk_collection, const ChunkKey &chunk_prefix)
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer * foreign_storage::ForeignStorageCache::getChunkBufferForPrecaching ( const ChunkKey chunk_key,
bool  is_new_buffer 
)

Definition at line 414 of file ForeignStorageCache.cpp.

References CHECK, File_Namespace::GlobalFileMgr::createBuffer(), File_Namespace::GlobalFileMgr::getBuffer(), and getGlobalFileMgr().

416  {
417  if (!is_new_buffer) {
418  CHECK(getGlobalFileMgr()->isBufferOnDevice(chunk_key));
419  return getGlobalFileMgr()->getBuffer(chunk_key);
420  } else {
421  CHECK(!getGlobalFileMgr()->isBufferOnDevice(chunk_key));
422  return getGlobalFileMgr()->createBuffer(chunk_key);
423  }
424 }
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:63
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:85
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

std::map< ChunkKey, AbstractBuffer * > foreign_storage::ForeignStorageCache::getChunkBuffersForCaching ( const std::vector< ChunkKey > &  chunk_keys) const

Definition at line 314 of file ForeignStorageCache.cpp.

References cached_chunks_, CHECK, CHECK_EQ, chunks_mutex_, DEBUG_TIMER, and global_file_mgr_.

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), and foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

315  {
316  auto timer = DEBUG_TIMER(__func__);
317  std::map<ChunkKey, AbstractBuffer*> chunk_buffer_map;
318  read_lock lock(chunks_mutex_);
319  for (const auto& chunk_key : chunk_keys) {
320  CHECK(cached_chunks_.find(chunk_key) == cached_chunks_.end());
321  CHECK(global_file_mgr_->isBufferOnDevice(chunk_key));
322  chunk_buffer_map[chunk_key] = global_file_mgr_->getBuffer(chunk_key);
323  CHECK(dynamic_cast<File_Namespace::FileBuffer*>(chunk_buffer_map[chunk_key]));
324  CHECK_EQ(chunk_buffer_map[chunk_key]->pageCount(), static_cast<size_t>(0));
325 
326  // Clear all buffer metadata
327  chunk_buffer_map[chunk_key]->resetToEmpty();
328  }
329  return chunk_buffer_map;
330 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the caller graph for this function:

File_Namespace::GlobalFileMgr* foreign_storage::ForeignStorageCache::getGlobalFileMgr ( ) const
inline

Definition at line 100 of file ForeignStorageCache.h.

Referenced by getCacheDirectoryForTablePrefix(), and getChunkBufferForPrecaching().

100  {
101  return global_file_mgr_.get();
102  }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_

+ Here is the caller graph for this function:

size_t foreign_storage::ForeignStorageCache::getNumCachedChunks ( ) const
inline

Definition at line 91 of file ForeignStorageCache.h.

91 { return cached_chunks_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumCachedMetadata ( ) const
inline

Definition at line 92 of file ForeignStorageCache.h.

92 { return cached_metadata_.size(); }
size_t foreign_storage::ForeignStorageCache::getNumChunksAdded ( ) const
inline

Definition at line 93 of file ForeignStorageCache.h.

size_t foreign_storage::ForeignStorageCache::getNumMetadataAdded ( ) const
inline

Definition at line 94 of file ForeignStorageCache.h.

bool foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix ( const ChunkKey chunk_prefix) const

Definition at line 244 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists(), foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

245  {
246  auto timer = DEBUG_TIMER(__func__);
248  // We don't use iterateOvermatchingPrefix() here because we want to exit early if
249  // possible.
250  ChunkKey upper_prefix(chunk_prefix);
251  upper_prefix.push_back(std::numeric_limits<int>::max());
252  auto end_it = cached_metadata_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
253  for (auto meta_it = cached_metadata_.lower_bound(chunk_prefix); meta_it != end_it;
254  ++meta_it) {
255  return true;
256  }
257  return false;
258 }
std::vector< int > ChunkKey
Definition: types.h:37
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::isMetadataCached ( const ChunkKey chunk_key) const

Definition at line 118 of file ForeignStorageCache.cpp.

References cached_metadata_, DEBUG_TIMER, and metadata_mutex_.

Referenced by foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment().

118  {
119  auto timer = DEBUG_TIMER(__func__);
121  return (cached_metadata_.find(chunk_key) != cached_metadata_.end());
122 }
mapd_shared_lock< mapd_shared_mutex > read_lock
#define DEBUG_TIMER(name)
Definition: Logger.h:313

+ Here is the caller graph for this function:

bool foreign_storage::ForeignStorageCache::recoverCacheForTable ( ChunkMetadataVector meta_vec,
const ChunkKey table_key 
)

Definition at line 124 of file ForeignStorageCache.cpp.

References cached_chunks_, cached_metadata_, CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, chunks_mutex_, global_file_mgr_, is_table_key(), is_varlen_data_key(), and is_varlen_key().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

125  {
127  CHECK(meta_vec.size() == 0);
128  CHECK(is_table_key(table_key));
129  CHECK(dynamic_cast<File_Namespace::FileMgr*>(global_file_mgr_->getFileMgr(table_key)));
130 
131  global_file_mgr_->getChunkMetadataVecForKeyPrefix(meta_vec, table_key);
132  for (auto& [chunk_key, metadata] : meta_vec) {
133  cached_metadata_.emplace(chunk_key);
134  // If there is no page count then the chunk was metadata only and should not be
135  // cached.
136  if (const auto& buf = global_file_mgr_->getBuffer(chunk_key); buf->pageCount() > 0) {
137  cached_chunks_.emplace(chunk_key);
138  }
139 
140  if (is_varlen_key(chunk_key)) {
141  // Metadata is only available for the data chunk, but look for the index as well
142  CHECK(is_varlen_data_key(chunk_key));
143  ChunkKey index_chunk_key = {chunk_key[CHUNK_KEY_DB_IDX],
144  chunk_key[CHUNK_KEY_TABLE_IDX],
145  chunk_key[CHUNK_KEY_COLUMN_IDX],
146  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
147  2};
148 
149  if (const auto& buf = global_file_mgr_->getBuffer(index_chunk_key);
150  buf->pageCount() > 0) {
151  cached_chunks_.emplace(index_chunk_key);
152  }
153  }
154  }
155  return (meta_vec.size() > 0);
156 }
std::vector< int > ChunkKey
Definition: types.h:37
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:65
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:61

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::ForeignStorageCache::validatePath ( const std::string &  base_path) const
private

Definition at line 372 of file ForeignStorageCache.cpp.

Referenced by ForeignStorageCache().

372  {
373  // check if base_path already exists, and if not create one
374  boost::filesystem::path path(base_path);
375  if (boost::filesystem::exists(path)) {
376  if (!boost::filesystem::is_directory(path)) {
377  throw std::runtime_error{
378  "cache path \"" + base_path +
379  "\" is not a directory. Please specify a valid directory "
380  "with --disk_cache_path=<path>, or use the default location."};
381  }
382  } else { // data directory does not exist
383  if (!boost::filesystem::create_directory(path)) {
384  throw std::runtime_error{
385  "could not create directory at cache path \"" + base_path +
386  "\". Please specify a valid directory location "
387  "with --disk_cache_path=<path> or use the default location."};
388  }
389  }
390 }

+ Here is the caller graph for this function:

Member Data Documentation

size_t foreign_storage::ForeignStorageCache::num_chunks_added_
private

Definition at line 126 of file ForeignStorageCache.h.

Referenced by cacheChunk(), and cacheTableChunks().

size_t foreign_storage::ForeignStorageCache::num_metadata_added_
private

Definition at line 127 of file ForeignStorageCache.h.

Referenced by cacheMetadataVec().


The documentation for this class was generated from the following files: