OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
PersistentStorageMgr Class Reference

#include <PersistentStorageMgr.h>

+ Inheritance diagram for PersistentStorageMgr:
+ Collaboration diagram for PersistentStorageMgr:

Public Member Functions

 PersistentStorageMgr (const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)
 
AbstractBuffercreateBuffer (const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override
 
void deleteBuffer (const ChunkKey &chunk_key, const bool purge) override
 
void deleteBuffersWithPrefix (const ChunkKey &chunk_key_prefix, const bool purge) override
 
AbstractBuffergetBuffer (const ChunkKey &chunk_key, const size_t num_bytes) override
 
void fetchBuffer (const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override
 
AbstractBufferputBuffer (const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
 
bool isBufferOnDevice (const ChunkKey &chunk_key) override
 
std::string printSlabs () override
 
void clearSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void checkpoint () override
 
void checkpoint (const int db_id, const int tb_id) override
 
AbstractBufferalloc (const size_t num_bytes) override
 
void free (AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 
std::string getStringMgrType () override
 
size_t getNumChunks () override
 
void removeTableRelatedDS (const int db_id, const int table_id) override
 
File_Namespace::GlobalFileMgrgetGlobalFileMgr () const
 
foreign_storage::ForeignStorageMgrgetForeignStorageMgr () const
 
foreign_storage::ForeignStorageCachegetDiskCache () const
 
const DiskCacheConfig getDiskCacheConfig () const
 

Static Public Member Functions

static PersistentStorageMgrcreatePersistentStorageMgr (const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)
 

Protected Member Functions

bool isForeignStorage (const ChunkKey &chunk_key) const
 
AbstractBufferMgrgetStorageMgrForTableKey (const ChunkKey &table_key) const
 
bool isChunkPrefixCacheable (const ChunkKey &chunk_prefix) const
 
int recoverDataWrapperIfCachedAndGetHighestFragId (const ChunkKey &table_key)
 

Protected Attributes

std::unique_ptr
< File_Namespace::GlobalFileMgr
global_file_mgr_
 
std::unique_ptr
< foreign_storage::ForeignStorageMgr
foreign_storage_mgr_
 
std::unique_ptr
< foreign_storage::ForeignStorageCache
disk_cache_
 
DiskCacheConfig disk_cache_config_
 

Detailed Description

Definition at line 26 of file PersistentStorageMgr.h.

Constructor & Destructor Documentation

PersistentStorageMgr::PersistentStorageMgr ( const std::string &  data_dir,
const size_t  num_reader_threads,
const DiskCacheConfig disk_cache_config 
)

Definition at line 34 of file PersistentStorageMgr.cpp.

References disk_cache_, disk_cache_config_, foreign_storage_mgr_, DiskCacheConfig::isEnabled(), and DiskCacheConfig::isEnabledForFSI().

Referenced by createPersistentStorageMgr().

37  : AbstractBufferMgr(0)
39  std::make_unique<File_Namespace::GlobalFileMgr>(0,
40  data_dir,
41  num_reader_threads))
42  , disk_cache_config_(disk_cache_config) {
43  disk_cache_ =
45  ? std::make_unique<foreign_storage::ForeignStorageCache>(disk_cache_config)
46  : nullptr;
49  ? std::make_unique<foreign_storage::CachingForeignStorageMgr>(disk_cache_.get())
50  : std::make_unique<foreign_storage::ForeignStorageMgr>();
51 }
bool isEnabledForFSI() const
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
DiskCacheConfig disk_cache_config_
bool isEnabled() const
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
std::unique_ptr< foreign_storage::ForeignStorageMgr > foreign_storage_mgr_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Function Documentation

AbstractBuffer * PersistentStorageMgr::alloc ( const size_t  num_bytes)
override

Definition at line 166 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

166  {
167  return global_file_mgr_->alloc(num_bytes);
168 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
void PersistentStorageMgr::checkpoint ( )
override

Definition at line 158 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

158  {
159  global_file_mgr_->checkpoint();
160 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
void PersistentStorageMgr::checkpoint ( const int  db_id,
const int  tb_id 
)
override

Definition at line 162 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

162  {
163  global_file_mgr_->checkpoint(db_id, tb_id);
164 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
void PersistentStorageMgr::clearSlabs ( )
override

Definition at line 138 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

138  {
139  global_file_mgr_->clearSlabs();
140 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
AbstractBuffer * PersistentStorageMgr::createBuffer ( const ChunkKey chunk_key,
const size_t  page_size,
const size_t  initial_size 
)
override

Definition at line 53 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::createBuffer().

55  {
56  return getStorageMgrForTableKey(chunk_key)->createBuffer(
57  chunk_key, page_size, initial_size);
58 }
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

PersistentStorageMgr * PersistentStorageMgr::createPersistentStorageMgr ( const std::string &  data_dir,
const size_t  num_reader_threads,
const DiskCacheConfig disk_cache_config 
)
static

Definition at line 23 of file PersistentStorageMgr.cpp.

References DiskCacheConfig::isEnabledForMutableTables(), and PersistentStorageMgr().

Referenced by Data_Namespace::DataMgr::populateMgrs().

26  {
27  if (config.isEnabledForMutableTables()) {
28  return new MutableCachePersistentStorageMgr(data_dir, num_reader_threads, config);
29  } else {
30  return new PersistentStorageMgr(data_dir, num_reader_threads, config);
31  }
32 }
PersistentStorageMgr(const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PersistentStorageMgr::deleteBuffer ( const ChunkKey chunk_key,
const bool  purge 
)
override

Definition at line 60 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::deleteBuffer().

60  {
61  getStorageMgrForTableKey(chunk_key)->deleteBuffer(chunk_key, purge);
62 }
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PersistentStorageMgr::deleteBuffersWithPrefix ( const ChunkKey chunk_key_prefix,
const bool  purge 
)
override

Definition at line 64 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::deleteBuffersWithPrefix().

65  {
66  getStorageMgrForTableKey(chunk_key_prefix)
67  ->deleteBuffersWithPrefix(chunk_key_prefix, purge);
68 }
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void PersistentStorageMgr::fetchBuffer ( const ChunkKey chunk_key,
AbstractBuffer destination_buffer,
const size_t  num_bytes 
)
override

Definition at line 75 of file PersistentStorageMgr.cpp.

References Data_Namespace::AbstractBuffer::copyTo(), disk_cache_, getStorageMgrForTableKey(), isChunkPrefixCacheable(), and isForeignStorage().

77  {
78  AbstractBufferMgr* mgr = getStorageMgrForTableKey(chunk_key);
79  if (isChunkPrefixCacheable(chunk_key)) {
80  AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
81  if (buffer) {
82  buffer->copyTo(destination_buffer, num_bytes);
83  return;
84  } else {
85  mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
86  if (!isForeignStorage(chunk_key)) {
87  // Foreign storage will read into cache buffers directly if enabled, so we do
88  // not want to cache foreign table chunks here as they will already be cached.
89  disk_cache_->cacheChunk(chunk_key, destination_buffer);
90  }
91  return;
92  }
93  }
94  mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
95 }
bool isForeignStorage(const ChunkKey &chunk_key) const
An AbstractBuffer is a unit of data management for a data manager.
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
void copyTo(AbstractBuffer *destination_buffer, const size_t num_bytes=0)
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

void PersistentStorageMgr::free ( AbstractBuffer buffer)
override

Definition at line 170 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

170  {
171  global_file_mgr_->free(buffer);
172 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
size_t PersistentStorageMgr::getAllocated ( )
override

Definition at line 150 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

150  {
151  return global_file_mgr_->getAllocated();
152 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
AbstractBuffer * PersistentStorageMgr::getBuffer ( const ChunkKey chunk_key,
const size_t  num_bytes 
)
override

Definition at line 70 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

71  {
72  return getStorageMgrForTableKey(chunk_key)->getBuffer(chunk_key, num_bytes);
73 }
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

void PersistentStorageMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunk_metadata,
const ChunkKey chunk_key_prefix 
)
override

Definition at line 104 of file PersistentStorageMgr.cpp.

References CHECK, disk_cache_, get_table_key(), getStorageMgrForTableKey(), has_table_prefix(), and isChunkPrefixCacheable().

106  {
107  CHECK(has_table_prefix(keyPrefix));
108  // If the disk has any cached metadata for a prefix then it is guaranteed to have all
109  // metadata for that table, so we can return a complete set. If it has no metadata,
110  // then it may be that the table has no data, or that it's just not cached, so we need
111  // to go to storage to check.
112  if (isChunkPrefixCacheable(keyPrefix)) {
113  if (disk_cache_->hasCachedMetadataForKeyPrefix(keyPrefix)) {
114  disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, keyPrefix);
115  return;
116  } else { // if we have no cached data attempt a recovery.
117  if (disk_cache_->recoverCacheForTable(chunk_metadata, get_table_key(keyPrefix))) {
118  return;
119  }
120  }
121  getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
122  keyPrefix);
123  disk_cache_->cacheMetadataVec(chunk_metadata);
124  } else {
125  getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
126  keyPrefix);
127  }
128 }
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:48
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
#define CHECK(condition)
Definition: Logger.h:197
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

foreign_storage::ForeignStorageCache * PersistentStorageMgr::getDiskCache ( ) const

Definition at line 222 of file PersistentStorageMgr.cpp.

References disk_cache_.

222  {
223  return disk_cache_ ? disk_cache_.get() : nullptr;
224 }
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
const DiskCacheConfig PersistentStorageMgr::getDiskCacheConfig ( ) const
inline

Definition at line 71 of file PersistentStorageMgr.h.

71 { return disk_cache_config_; }
DiskCacheConfig disk_cache_config_
foreign_storage::ForeignStorageMgr * PersistentStorageMgr::getForeignStorageMgr ( ) const

Definition at line 218 of file PersistentStorageMgr.cpp.

References foreign_storage_mgr_.

218  {
219  return foreign_storage_mgr_.get();
220 }
std::unique_ptr< foreign_storage::ForeignStorageMgr > foreign_storage_mgr_
File_Namespace::GlobalFileMgr * PersistentStorageMgr::getGlobalFileMgr ( ) const

Definition at line 186 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

186  {
187  return global_file_mgr_.get();
188 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
size_t PersistentStorageMgr::getInUseSize ( )
override

Definition at line 146 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

146  {
147  return global_file_mgr_->getInUseSize();
148 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
size_t PersistentStorageMgr::getMaxSize ( )
override

Definition at line 142 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

142  {
143  return global_file_mgr_->getMaxSize();
144 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
MgrType PersistentStorageMgr::getMgrType ( )
override

Definition at line 174 of file PersistentStorageMgr.cpp.

174  {
175  return PERSISTENT_STORAGE_MGR;
176 }
size_t PersistentStorageMgr::getNumChunks ( )
override

Definition at line 182 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

182  {
183  return global_file_mgr_->getNumChunks();
184 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
AbstractBufferMgr * PersistentStorageMgr::getStorageMgrForTableKey ( const ChunkKey table_key) const
protected

Definition at line 209 of file PersistentStorageMgr.cpp.

References foreign_storage_mgr_, global_file_mgr_, and isForeignStorage().

Referenced by createBuffer(), deleteBuffer(), deleteBuffersWithPrefix(), fetchBuffer(), getBuffer(), getChunkMetadataVecForKeyPrefix(), putBuffer(), and removeTableRelatedDS().

210  {
211  if (isForeignStorage(table_key)) {
212  return foreign_storage_mgr_.get();
213  } else {
214  return global_file_mgr_.get();
215  }
216 }
bool isForeignStorage(const ChunkKey &chunk_key) const
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
std::unique_ptr< foreign_storage::ForeignStorageMgr > foreign_storage_mgr_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string PersistentStorageMgr::getStringMgrType ( )
override

Definition at line 178 of file PersistentStorageMgr.cpp.

178  {
179  return ToString(PERSISTENT_STORAGE_MGR);
180 }
bool PersistentStorageMgr::isAllocationCapped ( )
override

Definition at line 154 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

154  {
155  return global_file_mgr_->isAllocationCapped();
156 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
bool PersistentStorageMgr::isBufferOnDevice ( const ChunkKey chunk_key)
override

Definition at line 130 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

130  {
131  return global_file_mgr_->isBufferOnDevice(chunk_key);
132 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
bool PersistentStorageMgr::isChunkPrefixCacheable ( const ChunkKey chunk_prefix) const
protected

Definition at line 226 of file PersistentStorageMgr.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, disk_cache_config_, has_table_prefix(), DiskCacheConfig::isEnabledForFSI(), DiskCacheConfig::isEnabledForMutableTables(), isForeignStorage(), and ForeignStorageInterface::lookupBufferManager().

Referenced by MutableCachePersistentStorageMgr::createBuffer(), fetchBuffer(), getChunkMetadataVecForKeyPrefix(), and removeTableRelatedDS().

226  {
227  CHECK(has_table_prefix(chunk_prefix));
228  // If this is an Arrow FSI table then we can't cache it.
230  chunk_prefix[CHUNK_KEY_TABLE_IDX])) {
231  return false;
232  }
234  !isForeignStorage(chunk_prefix)) ||
236 }
bool isEnabledForMutableTables() const
static Data_Namespace::AbstractBufferMgr * lookupBufferManager(const int db_id, const int table_id)
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
bool isEnabledForFSI() const
bool isForeignStorage(const ChunkKey &chunk_key) const
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:48
DiskCacheConfig disk_cache_config_
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool PersistentStorageMgr::isForeignStorage ( const ChunkKey chunk_key) const
protected

Definition at line 198 of file PersistentStorageMgr.cpp.

References CHECK, Catalog_Namespace::Catalog::checkedGet(), CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, StorageType::FOREIGN_TABLE, and has_table_prefix().

Referenced by MutableCachePersistentStorageMgr::deleteBuffer(), fetchBuffer(), getStorageMgrForTableKey(), and isChunkPrefixCacheable().

198  {
199  CHECK(has_table_prefix(chunk_key));
200  auto db_id = chunk_key[CHUNK_KEY_DB_IDX];
201  auto table_id = chunk_key[CHUNK_KEY_TABLE_IDX];
202  auto catalog = Catalog_Namespace::Catalog::checkedGet(db_id);
203 
204  auto table = catalog->getMetadataForTableImpl(table_id, false);
205  CHECK(table);
206  return table->storageType == StorageType::FOREIGN_TABLE;
207 }
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:48
static std::shared_ptr< Catalog > checkedGet(const int32_t db_id)
Definition: Catalog.cpp:3846
#define CHECK(condition)
Definition: Logger.h:197
static constexpr char const * FOREIGN_TABLE

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string PersistentStorageMgr::printSlabs ( )
override

Definition at line 134 of file PersistentStorageMgr.cpp.

References global_file_mgr_.

134  {
135  return global_file_mgr_->printSlabs();
136 }
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
AbstractBuffer * PersistentStorageMgr::putBuffer ( const ChunkKey chunk_key,
AbstractBuffer source_buffer,
const size_t  num_bytes 
)
override

Definition at line 97 of file PersistentStorageMgr.cpp.

References getStorageMgrForTableKey().

Referenced by MutableCachePersistentStorageMgr::putBuffer().

99  {
100  return getStorageMgrForTableKey(chunk_key)->putBuffer(
101  chunk_key, source_buffer, num_bytes);
102 }
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int PersistentStorageMgr::recoverDataWrapperIfCachedAndGetHighestFragId ( const ChunkKey table_key)
protected
void PersistentStorageMgr::removeTableRelatedDS ( const int  db_id,
const int  table_id 
)
override

Definition at line 190 of file PersistentStorageMgr.cpp.

References disk_cache_, getStorageMgrForTableKey(), and isChunkPrefixCacheable().

Referenced by MutableCachePersistentStorageMgr::removeTableRelatedDS().

190  {
191  const ChunkKey table_key{db_id, table_id};
192  if (isChunkPrefixCacheable(table_key)) {
193  disk_cache_->clearForTablePrefix(table_key);
194  }
195  getStorageMgrForTableKey(table_key)->removeTableRelatedDS(db_id, table_id);
196 }
std::vector< int > ChunkKey
Definition: types.h:37
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

DiskCacheConfig PersistentStorageMgr::disk_cache_config_
protected
std::unique_ptr<foreign_storage::ForeignStorageMgr> PersistentStorageMgr::foreign_storage_mgr_
protected

The documentation for this class was generated from the following files: