OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
File_Namespace::CachingGlobalFileMgr Class Reference

#include <CachingGlobalFileMgr.h>

+ Inheritance diagram for File_Namespace::CachingGlobalFileMgr:
+ Collaboration diagram for File_Namespace::CachingGlobalFileMgr:

Public Member Functions

 CachingGlobalFileMgr (int32_t device_id, std::shared_ptr< ForeignStorageInterface > fsi, const std::string &base_path, size_t num_reader_threads, foreign_storage::ForeignStorageCache *disk_cache, size_t defaultPageSize=DEFAULT_PAGE_SIZE)
 
AbstractBuffercreateBuffer (const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override
 
void deleteBuffer (const ChunkKey &chunk_key, const bool purge) override
 
void deleteBuffersWithPrefix (const ChunkKey &chunk_key_prefix, const bool purge) override
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &keyPrefix) override
 
void fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override
 
AbstractBufferputBuffer (const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override
 
void checkpoint () override
 
void checkpoint (const int db_id, const int tb_id) override
 
void removeTableRelatedDS (const int db_id, const int table_id) override
 
- Public Member Functions inherited from File_Namespace::GlobalFileMgr
 GlobalFileMgr (const int32_t deviceId, std::shared_ptr< ForeignStorageInterface > fsi, std::string basePath=".", const size_t num_reader_threads=0, const size_t defaultPageSize=DEFAULT_PAGE_SIZE)
 Constructor. More...
 
 ~GlobalFileMgr () override
 
AbstractBuffercreateBuffer (const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
 Creates a chunk with the specified key and page size. More...
 
bool isBufferOnDevice (const ChunkKey &key) override
 
void deleteBuffer (const ChunkKey &key, const bool purge=true) override
 Deletes the chunk with the specified key. More...
 
void deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
 
AbstractBuffergetBuffer (const ChunkKey &key, const size_t numBytes=0) override
 Returns the a pointer to the chunk with the specified key. More...
 
void fetchBuffer (const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
 
AbstractBufferputBuffer (const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
 Puts the contents of d into the Chunk with the given key. More...
 
AbstractBufferalloc (const size_t numBytes) override
 
void free (AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 
std::string getStringMgrType () override
 
std::string printSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void init ()
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
 
void checkpoint () override
 Fsyncs data files, writes out epoch and fsyncs that. More...
 
void checkpoint (const int32_t db_id, const int32_t tb_id) override
 
size_t getNumReaderThreads ()
 Returns number of threads defined by parameter num-reader-threads which should be used during initial load and consequent read of data. More...
 
size_t getNumChunks () override
 
void compactDataFiles (const int32_t db_id, const int32_t tb_id)
 
AbstractBufferMgrfindFileMgr (const int32_t db_id, const int32_t tb_id)
 
void setFileMgrParams (const int32_t db_id, const int32_t tb_id, const FileMgrParams &file_mgr_params)
 
AbstractBufferMgrgetFileMgr (const int32_t db_id, const int32_t tb_id)
 
AbstractBufferMgrgetFileMgr (const ChunkKey &key)
 
std::string getBasePath () const
 
size_t getDefaultPageSize () const
 
void writeFileMgrData (FileMgr *fileMgr=0)
 
int32_t getDBVersion () const
 
bool getDBConvert () const
 
void setDBConvert (bool val)
 
void removeTableRelatedDS (const int32_t db_id, const int32_t tb_id) override
 
void setTableEpoch (const int32_t db_id, const int32_t tb_id, const int32_t start_epoch)
 
size_t getTableEpoch (const int32_t db_id, const int32_t tb_id)
 
StorageStats getStorageStats (const int32_t db_id, const int32_t tb_id)
 
std::shared_ptr< FileMgrgetSharedFileMgr (const int db_id, const int table_id)
 
void setFileMgr (const int db_id, const int table_id, std::shared_ptr< FileMgr > file_mgr)
 
void closeFileMgr (const int32_t db_id, const int32_t tb_id)
 

Private Member Functions

bool isChunkPrefixCacheable (const ChunkKey &chunk_prefix) const
 

Private Attributes

foreign_storage::ForeignStorageCachedisk_cache_
 
std::set< ChunkKeycached_chunk_keys_
 

Additional Inherited Members

- Protected Attributes inherited from File_Namespace::GlobalFileMgr
std::shared_ptr
< ForeignStorageInterface
fsi_
 

Detailed Description

Definition at line 26 of file CachingGlobalFileMgr.h.

Constructor & Destructor Documentation

File_Namespace::CachingGlobalFileMgr::CachingGlobalFileMgr ( int32_t  device_id,
std::shared_ptr< ForeignStorageInterface fsi,
const std::string &  base_path,
size_t  num_reader_threads,
foreign_storage::ForeignStorageCache disk_cache,
size_t  defaultPageSize = DEFAULT_PAGE_SIZE 
)

Definition at line 22 of file CachingGlobalFileMgr.cpp.

References CHECK, and disk_cache_.

29  : GlobalFileMgr(device_id, fsi, base_path, num_reader_threads, default_page_size)
30  , disk_cache_(disk_cache) {
32 }
GlobalFileMgr(const int32_t deviceId, std::shared_ptr< ForeignStorageInterface > fsi, std::string basePath=".", const size_t num_reader_threads=0, const size_t defaultPageSize=DEFAULT_PAGE_SIZE)
Constructor.
foreign_storage::ForeignStorageCache * disk_cache_
#define CHECK(condition)
Definition: Logger.h:209

Member Function Documentation

void File_Namespace::CachingGlobalFileMgr::checkpoint ( )
override

Definition at line 129 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, foreign_storage::ForeignStorageCache::checkpoint(), File_Namespace::GlobalFileMgr::checkpoint(), disk_cache_, File_Namespace::GlobalFileMgr::fetchBuffer(), get_table_prefix(), File_Namespace::GlobalFileMgr::getBuffer(), isChunkPrefixCacheable(), and foreign_storage::ForeignStorageCache::putBuffer().

129  {
130  std::set<File_Namespace::TablePair> tables_to_checkpoint;
131  for (auto& key : cached_chunk_keys_) {
132  if (isChunkPrefixCacheable(key) && GlobalFileMgr::getBuffer(key)->isDirty()) {
133  tables_to_checkpoint.emplace(get_table_prefix(key));
135  GlobalFileMgr::fetchBuffer(key, &temp_buf, 0);
136  disk_cache_->putBuffer(key, &temp_buf);
137  }
138  }
139  for (auto [db, tb] : tables_to_checkpoint) {
140  disk_cache_->checkpoint(db, tb);
141  }
143 }
foreign_storage::ForeignStorageCache * disk_cache_
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
void putBuffer(const ChunkKey &, AbstractBuffer *, const size_t numBytes=0)
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:90
std::pair< int, int > get_table_prefix(const ChunkKey &key)
Definition: types.h:58
void checkpoint(const int32_t db_id, const int32_t tb_id)
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:94

+ Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::checkpoint ( const int  db_id,
const int  tb_id 
)
override

Definition at line 145 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, foreign_storage::ForeignStorageCache::checkpoint(), File_Namespace::GlobalFileMgr::checkpoint(), disk_cache_, File_Namespace::GlobalFileMgr::fetchBuffer(), File_Namespace::GlobalFileMgr::getBuffer(), isChunkPrefixCacheable(), Data_Namespace::AbstractBuffer::isDirty(), and foreign_storage::ForeignStorageCache::putBuffer().

145  {
146  if (isChunkPrefixCacheable({db_id, tb_id})) {
147  bool need_checkpoint{false};
148  ChunkKey chunk_prefix{db_id, tb_id};
149  ChunkKey upper_prefix(chunk_prefix);
150  upper_prefix.push_back(std::numeric_limits<int>::max());
151  auto end_it =
152  cached_chunk_keys_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
153  for (auto&& chunk_key_it = cached_chunk_keys_.lower_bound(chunk_prefix);
154  chunk_key_it != end_it;
155  ++chunk_key_it) {
156  if (GlobalFileMgr::getBuffer(*chunk_key_it)->isDirty()) {
157  need_checkpoint = true;
159  GlobalFileMgr::fetchBuffer(*chunk_key_it, &temp_buf, 0);
160  disk_cache_->putBuffer(*chunk_key_it, &temp_buf);
161  }
162  }
163  if (need_checkpoint) {
164  disk_cache_->checkpoint(db_id, tb_id);
165  }
166  }
167  GlobalFileMgr::checkpoint(db_id, tb_id);
168 }
std::vector< int > ChunkKey
Definition: types.h:37
foreign_storage::ForeignStorageCache * disk_cache_
void checkpoint() override
Fsyncs data files, writes out epoch and fsyncs that.
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
void putBuffer(const ChunkKey &, AbstractBuffer *, const size_t numBytes=0)
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:90
void checkpoint(const int32_t db_id, const int32_t tb_id)
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:94

+ Here is the call graph for this function:

AbstractBuffer * File_Namespace::CachingGlobalFileMgr::createBuffer ( const ChunkKey chunk_key,
const size_t  page_size,
const size_t  initial_size 
)
override

Definition at line 34 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, File_Namespace::GlobalFileMgr::createBuffer(), and isChunkPrefixCacheable().

36  {
37  auto buf = GlobalFileMgr::createBuffer(chunk_key, page_size, initial_size);
38  if (isChunkPrefixCacheable(chunk_key)) {
39  cached_chunk_keys_.emplace(chunk_key);
40  }
41  return buf;
42 }
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:68
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const

+ Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::deleteBuffer ( const ChunkKey chunk_key,
const bool  purge 
)
override

Definition at line 44 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, File_Namespace::GlobalFileMgr::deleteBuffer(), foreign_storage::ForeignStorageCache::deleteBufferIfExists(), disk_cache_, and isChunkPrefixCacheable().

44  {
45  if (isChunkPrefixCacheable(chunk_key)) {
47  cached_chunk_keys_.erase(chunk_key);
48  }
49  GlobalFileMgr::deleteBuffer(chunk_key, purge);
50 }
foreign_storage::ForeignStorageCache * disk_cache_
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:82
void deleteBufferIfExists(const ChunkKey &chunk_key)

+ Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::deleteBuffersWithPrefix ( const ChunkKey chunk_key_prefix,
const bool  purge 
)
override

Definition at line 52 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, CHECK, foreign_storage::ForeignStorageCache::clearForTablePrefix(), File_Namespace::GlobalFileMgr::deleteBuffersWithPrefix(), disk_cache_, get_table_key(), has_table_prefix(), and isChunkPrefixCacheable().

53  {
54  if (isChunkPrefixCacheable(chunk_key_prefix)) {
55  CHECK(has_table_prefix(chunk_key_prefix));
56  disk_cache_->clearForTablePrefix(get_table_key(chunk_key_prefix));
57 
58  ChunkKey upper_prefix(chunk_key_prefix);
59  upper_prefix.push_back(std::numeric_limits<int>::max());
60  auto end_it =
61  cached_chunk_keys_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
62  for (auto&& chunk_key_it = cached_chunk_keys_.lower_bound(chunk_key_prefix);
63  chunk_key_it != end_it;) {
64  chunk_key_it = cached_chunk_keys_.erase(chunk_key_it);
65  }
66  }
67  GlobalFileMgr::deleteBuffersWithPrefix(chunk_key_prefix, purge);
68 }
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
std::vector< int > ChunkKey
Definition: types.h:37
foreign_storage::ForeignStorageCache * disk_cache_
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:53
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:49
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::fetchBuffer ( const ChunkKey chunk_key,
AbstractBuffer destination_buffer,
const size_t  num_bytes 
)
override

Definition at line 90 of file CachingGlobalFileMgr.cpp.

References CHECK, Data_Namespace::AbstractBuffer::copyTo(), disk_cache_, File_Namespace::GlobalFileMgr::fetchBuffer(), File_Namespace::GlobalFileMgr::findFileMgr(), get_table_prefix(), foreign_storage::ForeignStorageCache::getCachedChunkIfExists(), has_table_prefix(), isChunkPrefixCacheable(), and foreign_storage::ForeignStorageCache::putBuffer().

92  {
93  if (isChunkPrefixCacheable(chunk_key)) {
94  // If we are recovering after a shutdown, it is possible for there to be cached data
95  // without the file_mgr being initialized, so we need to check if the file_mgr exists.
96  CHECK(has_table_prefix(chunk_key));
97  auto [db, table_id] = get_table_prefix(chunk_key);
98  auto file_mgr = GlobalFileMgr::findFileMgr(db, table_id);
99  if (file_mgr && file_mgr->getBuffer(chunk_key)->isDirty()) {
100  // It is possible for the fragmenter to write data to a FileBuffer and then attempt
101  // to fetch that bufer without checkpointing. In that case the cache will not have
102  // been updated and the cached buffer will be out of date, so we need to fetch the
103  // storage buffer.
104  GlobalFileMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
105  } else {
106  AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
107  if (buffer) {
108  buffer->copyTo(destination_buffer, num_bytes);
109  } else {
110  GlobalFileMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
111  disk_cache_->putBuffer(chunk_key, destination_buffer, num_bytes);
112  }
113  }
114  } else {
115  GlobalFileMgr::fetchBuffer(chunk_key, destination_buffer, num_bytes);
116  }
117 }
foreign_storage::ForeignStorageCache * disk_cache_
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:49
An AbstractBuffer is a unit of data management for a data manager.
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
void putBuffer(const ChunkKey &, AbstractBuffer *, const size_t numBytes=0)
void copyTo(AbstractBuffer *destination_buffer, const size_t num_bytes=0)
std::pair< int, int > get_table_prefix(const ChunkKey &key)
Definition: types.h:58
#define CHECK(condition)
Definition: Logger.h:209
File_Namespace::FileBuffer * getCachedChunkIfExists(const ChunkKey &)
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:94

+ Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunk_metadata,
const ChunkKey keyPrefix 
)
override

Definition at line 70 of file CachingGlobalFileMgr.cpp.

References foreign_storage::ForeignStorageCache::cacheMetadataVec(), CHECK, disk_cache_, foreign_storage::ForeignStorageCache::getCachedMetadataVecForKeyPrefix(), File_Namespace::GlobalFileMgr::getChunkMetadataVecForKeyPrefix(), has_table_prefix(), foreign_storage::ForeignStorageCache::hasCachedMetadataForKeyPrefix(), and isChunkPrefixCacheable().

72  {
73  CHECK(has_table_prefix(key_prefix));
74  if (isChunkPrefixCacheable(key_prefix)) {
75  // If the disk has any cached metadata for a prefix then it is guaranteed to have all
76  // metadata for that table, so we can return a complete set. If it has no metadata,
77  // then it may be that the table has no data, or that it's just not cached, so we need
78  // to go to storage to check.
79  if (disk_cache_->hasCachedMetadataForKeyPrefix(key_prefix)) {
80  disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
81  return;
82  }
83  }
84  GlobalFileMgr::getChunkMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
85  if (isChunkPrefixCacheable(key_prefix)) {
86  disk_cache_->cacheMetadataVec(chunk_metadata);
87  }
88 }
foreign_storage::ForeignStorageCache * disk_cache_
void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector &, const ChunkKey &) const
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:49
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
void cacheMetadataVec(const ChunkMetadataVector &)
#define CHECK(condition)
Definition: Logger.h:209
bool hasCachedMetadataForKeyPrefix(const ChunkKey &) const

+ Here is the call graph for this function:

bool File_Namespace::CachingGlobalFileMgr::isChunkPrefixCacheable ( const ChunkKey chunk_prefix) const
private

Definition at line 186 of file CachingGlobalFileMgr.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, File_Namespace::GlobalFileMgr::fsi_, and has_table_prefix().

Referenced by checkpoint(), createBuffer(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), getChunkMetadataVecForKeyPrefix(), putBuffer(), and removeTableRelatedDS().

186  {
187  CHECK(has_table_prefix(chunk_prefix));
188  // If this is an Arrow FSI table then we can't cache it.
189  if (fsi_->lookupBufferManager(chunk_prefix[CHUNK_KEY_DB_IDX],
190  chunk_prefix[CHUNK_KEY_TABLE_IDX])) {
191  return false;
192  }
193  return true;
194 }
std::shared_ptr< ForeignStorageInterface > fsi_
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:49
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

AbstractBuffer * File_Namespace::CachingGlobalFileMgr::putBuffer ( const ChunkKey chunk_key,
AbstractBuffer source_buffer,
const size_t  num_bytes 
)
override

Definition at line 119 of file CachingGlobalFileMgr.cpp.

References disk_cache_, isChunkPrefixCacheable(), foreign_storage::ForeignStorageCache::putBuffer(), and File_Namespace::GlobalFileMgr::putBuffer().

121  {
122  auto buf = GlobalFileMgr::putBuffer(chunk_key, source_buffer, num_bytes);
123  if (isChunkPrefixCacheable(chunk_key)) {
124  disk_cache_->putBuffer(chunk_key, source_buffer, num_bytes);
125  }
126  return buf;
127 }
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
foreign_storage::ForeignStorageCache * disk_cache_
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
void putBuffer(const ChunkKey &, AbstractBuffer *, const size_t numBytes=0)

+ Here is the call graph for this function:

void File_Namespace::CachingGlobalFileMgr::removeTableRelatedDS ( const int  db_id,
const int  table_id 
)
override

Definition at line 170 of file CachingGlobalFileMgr.cpp.

References cached_chunk_keys_, foreign_storage::ForeignStorageCache::clearForTablePrefix(), disk_cache_, isChunkPrefixCacheable(), and File_Namespace::GlobalFileMgr::removeTableRelatedDS().

170  {
171  if (isChunkPrefixCacheable({db_id, table_id})) {
172  const ChunkKey table_key{db_id, table_id};
173  disk_cache_->clearForTablePrefix(table_key);
174  ChunkKey upper_prefix(table_key);
175  upper_prefix.push_back(std::numeric_limits<int>::max());
176  auto end_it =
177  cached_chunk_keys_.upper_bound(static_cast<const ChunkKey>(upper_prefix));
178  for (auto&& chunk_key_it = cached_chunk_keys_.lower_bound(table_key);
179  chunk_key_it != end_it;) {
180  chunk_key_it = cached_chunk_keys_.erase(chunk_key_it);
181  }
182  }
183  GlobalFileMgr::removeTableRelatedDS(db_id, table_id);
184 }
std::vector< int > ChunkKey
Definition: types.h:37
foreign_storage::ForeignStorageCache * disk_cache_
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
void removeTableRelatedDS(const int32_t db_id, const int32_t tb_id) override

+ Here is the call graph for this function:

Member Data Documentation

std::set<ChunkKey> File_Namespace::CachingGlobalFileMgr::cached_chunk_keys_
private

The documentation for this class was generated from the following files: