OmniSciDB  bf83d84833
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
foreign_storage::CachingForeignStorageMgr Class Reference

#include <CachingForeignStorageMgr.h>

+ Inheritance diagram for foreign_storage::CachingForeignStorageMgr:
+ Collaboration diagram for foreign_storage::CachingForeignStorageMgr:

Public Member Functions

 CachingForeignStorageMgr (ForeignStorageCache *cache)
 
void fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
 
void refreshTable (const ChunkKey &table_key, const bool evict_cached_entries) override
 
- Public Member Functions inherited from foreign_storage::ForeignStorageMgr
 ForeignStorageMgr ()
 
AbstractBuffercreateBuffer (const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override
 
void deleteBuffer (const ChunkKey &chunk_key, const bool purge) override
 
void deleteBuffersWithPrefix (const ChunkKey &chunk_key_prefix, const bool purge) override
 
AbstractBuffergetBuffer (const ChunkKey &chunk_key, const size_t num_bytes) override
 
void fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override
 
AbstractBufferputBuffer (const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
 
bool isBufferOnDevice (const ChunkKey &chunk_key) override
 
std::string printSlabs () override
 
void clearSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void checkpoint () override
 
void checkpoint (const int db_id, const int tb_id) override
 
AbstractBufferalloc (const size_t num_bytes) override
 
void free (AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 
std::string getStringMgrType () override
 
size_t getNumChunks () override
 
void removeTableRelatedDS (const int db_id, const int table_id) override
 
bool hasDataWrapperForChunk (const ChunkKey &chunk_key)
 
bool isDatawrapperRestored (const ChunkKey &chunk_key)
 
void setDataWrapper (const ChunkKey &table_key, std::shared_ptr< MockForeignDataWrapper > data_wrapper)
 
void setColumnHints (std::map< ChunkKey, std::vector< int >> &columns_per_table)
 

Private Member Functions

void recoverDataWrapperFromDisk (const ChunkKey &table_key, const ChunkMetadataVector &chunk_metadata)
 
void refreshTableInCache (const ChunkKey &table_key)
 
int getHighestCachedFragId (const ChunkKey &table_key)
 
void refreshAppendTableInCache (const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)
 
void refreshNonAppendTableInCache (const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)
 
void refreshChunksInCacheByFragment (const std::vector< ChunkKey > &old_chunk_keys, int last_frag_id)
 
void createOrRecoverDataWrapperIfNotExists (const ChunkKey &chunk_key)
 

Private Attributes

ForeignStorageCachedisk_cache_
 

Additional Inherited Members

- Protected Member Functions inherited from foreign_storage::ForeignStorageMgr
bool createDataWrapperIfNotExists (const ChunkKey &chunk_key)
 
std::shared_ptr
< ForeignDataWrapper
getDataWrapper (const ChunkKey &chunk_key)
 
bool fetchBufferIfTempBufferMapEntryExists (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes)
 
void createAndPopulateDataWrapperIfNotExists (const ChunkKey &chunk_key)
 
std::map< ChunkKey,
AbstractBuffer * > 
allocateTempBuffersForChunks (const std::set< ChunkKey > &chunk_keys)
 
void clearTempChunkBufferMapEntriesForTable (const ChunkKey &table_key)
 
void clearTempChunkBufferMapEntriesForTableUnlocked (const ChunkKey &table_key)
 
void getOptionalChunkKeySet (std::set< ChunkKey > &optional_chunk_keys, const ChunkKey &chunk_key, const std::set< ChunkKey > &required_chunk_keys)
 
- Static Protected Member Functions inherited from foreign_storage::ForeignStorageMgr
static void checkIfS3NeedsToBeEnabled (const ChunkKey &chunk_key)
 
- Protected Attributes inherited from foreign_storage::ForeignStorageMgr
std::shared_mutex data_wrapper_mutex_
 
std::map< ChunkKey,
std::shared_ptr
< ForeignDataWrapper > > 
data_wrapper_map_
 
std::map< ChunkKey,
std::unique_ptr
< AbstractBuffer > > 
temp_chunk_buffer_map_
 
std::shared_mutex temp_chunk_buffer_map_mutex_
 
std::shared_mutex columns_hints_mutex_
 
std::map< ChunkKey,
std::vector< int > > 
columns_hints_per_table_
 

Detailed Description

Definition at line 28 of file CachingForeignStorageMgr.h.

Constructor & Destructor Documentation

foreign_storage::CachingForeignStorageMgr::CachingForeignStorageMgr ( ForeignStorageCache cache)

Definition at line 31 of file CachingForeignStorageMgr.cpp.

References CHECK, and disk_cache_.

32  : ForeignStorageMgr(), disk_cache_(cache) {
34 }
#define CHECK(condition)
Definition: Logger.h:197

Member Function Documentation

void foreign_storage::CachingForeignStorageMgr::createOrRecoverDataWrapperIfNotExists ( const ChunkKey chunk_key)
private

Definition at line 252 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageMgr::createDataWrapperIfNotExists(), disk_cache_, get_table_key(), foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), foreign_storage::ForeignStorageMgr::getDataWrapper(), foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix(), and recoverDataWrapperFromDisk().

Referenced by fetchBuffer(), and refreshAppendTableInCache().

253  {
254  ChunkKey table_key = get_table_key(chunk_key);
255  if (createDataWrapperIfNotExists(table_key)) {
256  ChunkMetadataVector chunk_metadata;
257  if (disk_cache_->hasCachedMetadataForKeyPrefix(table_key)) {
258  disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, table_key);
259  recoverDataWrapperFromDisk(table_key, chunk_metadata);
260  } else {
261  getDataWrapper(table_key)->populateChunkMetadata(chunk_metadata);
262  }
263  }
264 }
std::vector< int > ChunkKey
Definition: types.h:37
std::shared_ptr< ForeignDataWrapper > getDataWrapper(const ChunkKey &chunk_key)
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector &, const ChunkKey &) const
void recoverDataWrapperFromDisk(const ChunkKey &table_key, const ChunkMetadataVector &chunk_metadata)
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
bool hasCachedMetadataForKeyPrefix(const ChunkKey &) const
bool createDataWrapperIfNotExists(const ChunkKey &chunk_key)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::fetchBuffer ( const ChunkKey chunk_key,
AbstractBuffer destination_buffer,
const size_t  num_bytes 
)
override

Definition at line 36 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheTableChunks(), CHECK, Catalog_Namespace::SysCatalog::checkedGetCatalog(), CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, Data_Namespace::AbstractBuffer::copyTo(), createOrRecoverDataWrapperIfNotExists(), foreign_storage::DataWrapperType::CSV, disk_cache_, foreign_storage::get_keys_set_from_table(), foreign_storage::get_keys_vec_from_table(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), foreign_storage::ForeignStorageCache::getChunkBuffersForCaching(), foreign_storage::ForeignStorageMgr::getDataWrapper(), foreign_storage::ForeignStorageMgr::getOptionalChunkKeySet(), Catalog_Namespace::SysCatalog::instance(), and Data_Namespace::AbstractBuffer::isDirty().

38  {
39  CHECK(destination_buffer);
40  CHECK(!destination_buffer->isDirty());
41 
43 
44  // TODO: Populate optional buffers as part of CSV performance improvement
45  std::vector<ChunkKey> chunk_keys = get_keys_vec_from_table(chunk_key);
46  std::vector<ChunkKey> optional_chunk_keys;
47  std::map<ChunkKey, AbstractBuffer*> optional_buffers;
48 
49  // Use hints to prefetch other chunks in fragment into cache
51  chunk_key[CHUNK_KEY_DB_IDX]);
52  auto foreign_table = catalog->getForeignTableUnlocked(chunk_key[CHUNK_KEY_TABLE_IDX]);
53  if (foreign_table->foreign_server->data_wrapper_type ==
54  foreign_storage::DataWrapperType::CSV) // optimization only useful for column based
55  // formats
56  {
57  std::set<ChunkKey> optional_chunk_key_set;
59  optional_chunk_key_set, chunk_key, get_keys_set_from_table(chunk_key));
60  for (const auto& key : optional_chunk_key_set) {
61  if (disk_cache_->getCachedChunkIfExists(key) == nullptr) {
62  optional_chunk_keys.emplace_back(key);
63  }
64  }
65 
66  if (optional_chunk_keys.size()) {
67  optional_buffers = disk_cache_->getChunkBuffersForCaching(optional_chunk_keys);
68  }
69  }
70 
71  std::map<ChunkKey, AbstractBuffer*> required_buffers =
73  CHECK(required_buffers.find(chunk_key) != required_buffers.end());
74  getDataWrapper(chunk_key)->populateChunkBuffers(required_buffers, optional_buffers);
75  disk_cache_->cacheTableChunks(chunk_keys);
76  if (optional_chunk_keys.size()) {
77  disk_cache_->cacheTableChunks(optional_chunk_keys);
78  }
79 
80  AbstractBuffer* buffer = required_buffers.at(chunk_key);
81  CHECK(buffer);
82 
83  buffer->copyTo(destination_buffer, num_bytes);
84 }
void getOptionalChunkKeySet(std::set< ChunkKey > &optional_chunk_keys, const ChunkKey &chunk_key, const std::set< ChunkKey > &required_chunk_keys)
std::vector< ChunkKey > get_keys_vec_from_table(const ChunkKey &destination_chunk_key)
std::shared_ptr< ForeignDataWrapper > getDataWrapper(const ChunkKey &chunk_key)
std::set< ChunkKey > get_keys_set_from_table(const ChunkKey &destination_chunk_key)
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
std::map< ChunkKey, AbstractBuffer * > getChunkBuffersForCaching(const std::vector< ChunkKey > &chunk_keys) const
void cacheTableChunks(const std::vector< ChunkKey > &chunk_keys)
AbstractBuffer * getCachedChunkIfExists(const ChunkKey &)
static SysCatalog & instance()
Definition: SysCatalog.h:288
void createOrRecoverDataWrapperIfNotExists(const ChunkKey &chunk_key)
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
An AbstractBuffer is a unit of data management for a data manager.
std::shared_ptr< Catalog > checkedGetCatalog(const int32_t db_id)
void copyTo(AbstractBuffer *destination_buffer, const size_t num_bytes=0)
#define CHECK(condition)
Definition: Logger.h:197
static constexpr char const * CSV
Definition: ForeignServer.h:37

+ Here is the call graph for this function:

void foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunk_metadata,
const ChunkKey chunk_key_prefix 
)
override

Definition at line 86 of file CachingForeignStorageMgr.cpp.

References disk_cache_, foreign_storage::ForeignStorageCache::getCacheDirectoryForTablePrefix(), foreign_storage::ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), foreign_storage::ForeignStorageMgr::getDataWrapper(), and foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::wrapper_file_name.

Referenced by refreshAppendTableInCache(), and refreshNonAppendTableInCache().

88  {
89  ForeignStorageMgr::getChunkMetadataVecForKeyPrefix(chunk_metadata, keyPrefix);
90  getDataWrapper(keyPrefix)->serializeDataWrapperInternals(
92 }
std::shared_ptr< ForeignDataWrapper > getDataWrapper(const ChunkKey &chunk_key)
std::string getCacheDirectoryForTablePrefix(const ChunkKey &) const
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int foreign_storage::CachingForeignStorageMgr::getHighestCachedFragId ( const ChunkKey table_key)
private

Definition at line 134 of file CachingForeignStorageMgr.cpp.

References CHUNK_KEY_FRAGMENT_IDX, disk_cache_, foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), and foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix().

Referenced by refreshAppendTableInCache().

134  {
135  // Determine last fragment ID
136  int last_frag_id = 0;
137  if (disk_cache_->hasCachedMetadataForKeyPrefix(table_key)) {
138  ChunkMetadataVector cached_metadata;
139  disk_cache_->getCachedMetadataVecForKeyPrefix(cached_metadata, table_key);
140  for (const auto& [key, metadata] : cached_metadata) {
141  last_frag_id = std::max(last_frag_id, key[CHUNK_KEY_FRAGMENT_IDX]);
142  }
143  }
144  return last_frag_id;
145 }
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector &, const ChunkKey &) const
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
bool hasCachedMetadataForKeyPrefix(const ChunkKey &) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk ( const ChunkKey table_key,
const ChunkMetadataVector chunk_metadata 
)
private

Definition at line 94 of file CachingForeignStorageMgr.cpp.

References disk_cache_, foreign_storage::ForeignStorageCache::getCacheDirectoryForTablePrefix(), foreign_storage::ForeignStorageMgr::getDataWrapper(), and foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::wrapper_file_name.

Referenced by createOrRecoverDataWrapperIfNotExists().

96  {
97  getDataWrapper(table_key)->restoreDataWrapperInternals(
99  chunk_metadata);
100 }
std::shared_ptr< ForeignDataWrapper > getDataWrapper(const ChunkKey &chunk_key)
std::string getCacheDirectoryForTablePrefix(const ChunkKey &) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshAppendTableInCache ( const ChunkKey table_key,
const std::vector< ChunkKey > &  old_chunk_keys 
)
private

Definition at line 147 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataWithFragIdGreaterOrEqualTo(), CHECK, createOrRecoverDataWrapperIfNotExists(), disk_cache_, getChunkMetadataVecForKeyPrefix(), getHighestCachedFragId(), is_table_key(), and refreshChunksInCacheByFragment().

Referenced by refreshTableInCache().

149  {
150  CHECK(is_table_key(table_key));
152  int last_frag_id = getHighestCachedFragId(table_key);
153 
154  ChunkMetadataVector storage_metadata;
155  getChunkMetadataVecForKeyPrefix(storage_metadata, table_key);
156  try {
157  disk_cache_->cacheMetadataWithFragIdGreaterOrEqualTo(storage_metadata, last_frag_id);
158  refreshChunksInCacheByFragment(old_chunk_keys, last_frag_id);
159  } catch (std::runtime_error& e) {
161  }
162 }
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
void createOrRecoverDataWrapperIfNotExists(const ChunkKey &chunk_key)
void refreshChunksInCacheByFragment(const std::vector< ChunkKey > &old_chunk_keys, int last_frag_id)
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void cacheMetadataWithFragIdGreaterOrEqualTo(const ChunkMetadataVector &metadata_vec, const int frag_id)
#define CHECK(condition)
Definition: Logger.h:197
int getHighestCachedFragId(const ChunkKey &table_key)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshChunksInCacheByFragment ( const std::vector< ChunkKey > &  old_chunk_keys,
int  last_frag_id 
)
private

Definition at line 180 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheTableChunks(), CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, disk_cache_, get_table_key(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), foreign_storage::ForeignStorageCache::getChunkBuffersForCaching(), foreign_storage::ForeignStorageMgr::getDataWrapper(), is_varlen_data_key(), is_varlen_key(), foreign_storage::ForeignStorageCache::isMetadataCached(), LOG, foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::MAX_REFRESH_TIME_IN_SECONDS, and logger::WARNING.

Referenced by refreshAppendTableInCache(), and refreshNonAppendTableInCache().

182  {
183  int64_t total_time{0};
184  auto fragment_refresh_start_time = std::chrono::high_resolution_clock::now();
185 
186  if (old_chunk_keys.empty()) {
187  return;
188  }
189  // Iterate through previously cached chunks and re-cache them. Caching is
190  // done one fragment at a time, for all applicable chunks in the fragment.
191  std::map<ChunkKey, AbstractBuffer*> optional_buffers;
192  std::vector<ChunkKey> chunk_keys_to_be_cached;
193  auto fragment_id = old_chunk_keys[0][CHUNK_KEY_FRAGMENT_IDX];
194  const ChunkKey table_key{get_table_key(old_chunk_keys[0])};
195  std::vector<ChunkKey> chunk_keys_in_fragment;
196  for (const auto& chunk_key : old_chunk_keys) {
197  if (chunk_key[CHUNK_KEY_FRAGMENT_IDX] < start_frag_id) {
198  continue;
199  }
200  if (disk_cache_->isMetadataCached(chunk_key)) {
201  if (chunk_key[CHUNK_KEY_FRAGMENT_IDX] != fragment_id) {
202  if (chunk_keys_in_fragment.size() > 0) {
203  auto required_buffers =
204  disk_cache_->getChunkBuffersForCaching(chunk_keys_in_fragment);
205  getDataWrapper(table_key)->populateChunkBuffers(required_buffers,
206  optional_buffers);
207  chunk_keys_in_fragment.clear();
208  }
209  // At this point, cache buffers for refreshable chunks in the last fragment
210  // have been populated. Exit if the max refresh time has been exceeded.
211  // Otherwise, move to the next fragment.
212  auto current_time = std::chrono::high_resolution_clock::now();
213  total_time += std::chrono::duration_cast<std::chrono::seconds>(
214  current_time - fragment_refresh_start_time)
215  .count();
216  if (total_time >= MAX_REFRESH_TIME_IN_SECONDS) {
217  LOG(WARNING) << "Refresh time exceeded for table key: { " << table_key[0]
218  << ", " << table_key[1] << " } after fragment id: " << fragment_id;
219  break;
220  } else {
221  fragment_refresh_start_time = std::chrono::high_resolution_clock::now();
222  }
223  fragment_id = chunk_key[CHUNK_KEY_FRAGMENT_IDX];
224  }
225  // Key may have been cached during scan
226  if (disk_cache_->getCachedChunkIfExists(chunk_key) == nullptr) {
227  if (is_varlen_key(chunk_key)) {
228  CHECK(is_varlen_data_key(chunk_key));
229  ChunkKey index_chunk_key{chunk_key[CHUNK_KEY_DB_IDX],
230  chunk_key[CHUNK_KEY_TABLE_IDX],
231  chunk_key[CHUNK_KEY_COLUMN_IDX],
232  chunk_key[CHUNK_KEY_FRAGMENT_IDX],
233  2};
234  chunk_keys_in_fragment.emplace_back(index_chunk_key);
235  chunk_keys_to_be_cached.emplace_back(index_chunk_key);
236  }
237  chunk_keys_in_fragment.emplace_back(chunk_key);
238  chunk_keys_to_be_cached.emplace_back(chunk_key);
239  }
240  }
241  }
242  if (chunk_keys_in_fragment.size() > 0) {
243  auto required_buffers =
244  disk_cache_->getChunkBuffersForCaching(chunk_keys_in_fragment);
245  getDataWrapper(table_key)->populateChunkBuffers(required_buffers, optional_buffers);
246  }
247  if (chunk_keys_to_be_cached.size() > 0) {
248  disk_cache_->cacheTableChunks(chunk_keys_to_be_cached);
249  }
250 }
std::vector< int > ChunkKey
Definition: types.h:37
bool isMetadataCached(const ChunkKey &) const
std::shared_ptr< ForeignDataWrapper > getDataWrapper(const ChunkKey &chunk_key)
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:65
#define LOG(tag)
Definition: Logger.h:188
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
std::map< ChunkKey, AbstractBuffer * > getChunkBuffersForCaching(const std::vector< ChunkKey > &chunk_keys) const
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
void cacheTableChunks(const std::vector< ChunkKey > &chunk_keys)
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
AbstractBuffer * getCachedChunkIfExists(const ChunkKey &)
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
#define CHECK(condition)
Definition: Logger.h:197
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
bool is_varlen_key(const ChunkKey &key)
Definition: types.h:61

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshNonAppendTableInCache ( const ChunkKey table_key,
const std::vector< ChunkKey > &  old_chunk_keys 
)
private

Definition at line 164 of file CachingForeignStorageMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, foreign_storage::ForeignStorageCache::clearForTablePrefix(), disk_cache_, getChunkMetadataVecForKeyPrefix(), is_table_key(), and refreshChunksInCacheByFragment().

Referenced by refreshTableInCache().

166  {
167  CHECK(is_table_key(table_key));
168  ChunkMetadataVector storage_metadata;
169  disk_cache_->clearForTablePrefix(table_key);
170  getChunkMetadataVecForKeyPrefix(storage_metadata, table_key);
171 
172  try {
173  disk_cache_->cacheMetadataVec(storage_metadata);
174  refreshChunksInCacheByFragment(old_chunk_keys, 0);
175  } catch (std::runtime_error& e) {
177  }
178 }
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
void refreshChunksInCacheByFragment(const std::vector< ChunkKey > &old_chunk_keys, int last_frag_id)
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void cacheMetadataVec(const ChunkMetadataVector &)
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshTable ( const ChunkKey table_key,
const bool  evict_cached_entries 
)
overridevirtual

Reimplemented from foreign_storage::ForeignStorageMgr.

Definition at line 102 of file CachingForeignStorageMgr.cpp.

References CHECK, foreign_storage::ForeignStorageMgr::checkIfS3NeedsToBeEnabled(), foreign_storage::ForeignStorageCache::clearForTablePrefix(), foreign_storage::ForeignStorageMgr::clearTempChunkBufferMapEntriesForTable(), disk_cache_, is_table_key(), and refreshTableInCache().

103  {
104  CHECK(is_table_key(table_key));
107  evict_cached_entries ? disk_cache_->clearForTablePrefix(table_key)
108  : refreshTableInCache(table_key);
109 }
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
static void checkIfS3NeedsToBeEnabled(const ChunkKey &chunk_key)
void refreshTableInCache(const ChunkKey &table_key)
#define CHECK(condition)
Definition: Logger.h:197
void clearTempChunkBufferMapEntriesForTable(const ChunkKey &table_key)

+ Here is the call graph for this function:

void foreign_storage::CachingForeignStorageMgr::refreshTableInCache ( const ChunkKey table_key)
private

Definition at line 111 of file CachingForeignStorageMgr.cpp.

References CHECK, Catalog_Namespace::SysCatalog::checkedGetCatalog(), CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, disk_cache_, foreign_storage::ForeignStorageCache::getCachedChunksForKeyPrefix(), foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix(), Catalog_Namespace::SysCatalog::instance(), is_table_key(), foreign_storage::ForeignStorageCache::recoverCacheForTable(), refreshAppendTableInCache(), and refreshNonAppendTableInCache().

Referenced by refreshTable().

111  {
112  CHECK(is_table_key(table_key));
113 
114  // Before we can refresh a table we should make sure it has recovered any data
115  // if the table has been unused since the last server restart.
116  if (!disk_cache_->hasCachedMetadataForKeyPrefix(table_key)) {
117  ChunkMetadataVector old_cached_metadata;
118  disk_cache_->recoverCacheForTable(old_cached_metadata, table_key);
119  }
120 
121  // Preserve the list of which chunks were cached per table to refresh after clear.
122  std::vector<ChunkKey> old_chunk_keys =
124 
125  bool append_mode = Catalog_Namespace::SysCatalog::instance()
127  ->getForeignTableUnlocked(table_key[CHUNK_KEY_TABLE_IDX])
128  ->isAppendMode();
129 
130  append_mode ? refreshAppendTableInCache(table_key, old_chunk_keys)
131  : refreshNonAppendTableInCache(table_key, old_chunk_keys);
132 }
void refreshAppendTableInCache(const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)
bool is_table_key(const ChunkKey &key)
Definition: types.h:44
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
void refreshNonAppendTableInCache(const ChunkKey &table_key, const std::vector< ChunkKey > &old_chunk_keys)
static SysCatalog & instance()
Definition: SysCatalog.h:288
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
bool recoverCacheForTable(ChunkMetadataVector &, const ChunkKey &)
std::shared_ptr< Catalog > checkedGetCatalog(const int32_t db_id)
std::vector< ChunkKey > getCachedChunksForKeyPrefix(const ChunkKey &) const
#define CHECK(condition)
Definition: Logger.h:197
bool hasCachedMetadataForKeyPrefix(const ChunkKey &) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation


The documentation for this class was generated from the following files: