OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
PersistentStorageMgr.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "PersistentStorageMgr.h"
18 #include "Catalog/Catalog.h"
22 
24  const std::string& data_dir,
25  const size_t num_reader_threads,
26  const DiskCacheConfig& config) {
27  if (config.isEnabledForMutableTables()) {
28  return new MutableCachePersistentStorageMgr(data_dir, num_reader_threads, config);
29  } else {
30  return new PersistentStorageMgr(data_dir, num_reader_threads, config);
31  }
32 }
33 
34 PersistentStorageMgr::PersistentStorageMgr(const std::string& data_dir,
35  const size_t num_reader_threads,
36  const DiskCacheConfig& disk_cache_config)
37  : AbstractBufferMgr(0)
38  , global_file_mgr_(
39  std::make_unique<File_Namespace::GlobalFileMgr>(0,
40  data_dir,
41  num_reader_threads))
42  , disk_cache_config_(disk_cache_config) {
43  disk_cache_ =
45  ? std::make_unique<foreign_storage::ForeignStorageCache>(disk_cache_config)
46  : nullptr;
49  ? std::make_unique<foreign_storage::CachingForeignStorageMgr>(disk_cache_.get())
50  : std::make_unique<foreign_storage::ForeignStorageMgr>();
51 }
52 
54  const size_t page_size,
55  const size_t initial_size) {
56  return getStorageMgrForTableKey(chunk_key)->createBuffer(
57  chunk_key, page_size, initial_size);
58 }
59 
60 void PersistentStorageMgr::deleteBuffer(const ChunkKey& chunk_key, const bool purge) {
61  getStorageMgrForTableKey(chunk_key)->deleteBuffer(chunk_key, purge);
62 }
63 
65  const bool purge) {
66  getStorageMgrForTableKey(chunk_key_prefix)
67  ->deleteBuffersWithPrefix(chunk_key_prefix, purge);
68 }
69 
71  const size_t num_bytes) {
72  return getStorageMgrForTableKey(chunk_key)->getBuffer(chunk_key, num_bytes);
73 }
74 
76  AbstractBuffer* destination_buffer,
77  const size_t num_bytes) {
78  AbstractBufferMgr* mgr = getStorageMgrForTableKey(chunk_key);
79  if (isChunkPrefixCacheable(chunk_key)) {
80  AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
81  if (buffer) {
82  buffer->copyTo(destination_buffer, num_bytes);
83  return;
84  } else {
85  mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
86  if (!isForeignStorage(chunk_key)) {
87  // Foreign storage will read into cache buffers directly if enabled, so we do
88  // not want to cache foreign table chunks here as they will already be cached.
89  disk_cache_->cacheChunk(chunk_key, destination_buffer);
90  }
91  return;
92  }
93  }
94  mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
95 }
96 
98  AbstractBuffer* source_buffer,
99  const size_t num_bytes) {
100  return getStorageMgrForTableKey(chunk_key)->putBuffer(
101  chunk_key, source_buffer, num_bytes);
102 }
103 
105  ChunkMetadataVector& chunk_metadata,
106  const ChunkKey& keyPrefix) {
107  CHECK(has_table_prefix(keyPrefix));
108  // If the disk has any cached metadata for a prefix then it is guaranteed to have all
109  // metadata for that table, so we can return a complete set. If it has no metadata,
110  // then it may be that the table has no data, or that it's just not cached, so we need
111  // to go to storage to check.
112  if (isChunkPrefixCacheable(keyPrefix)) {
113  if (disk_cache_->hasCachedMetadataForKeyPrefix(keyPrefix)) {
114  disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, keyPrefix);
115  return;
116  } else { // if we have no cached data attempt a recovery.
117  if (disk_cache_->recoverCacheForTable(chunk_metadata, get_table_key(keyPrefix))) {
118  return;
119  }
120  }
121  getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
122  keyPrefix);
123  disk_cache_->cacheMetadataVec(chunk_metadata);
124  } else {
125  getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
126  keyPrefix);
127  }
128 }
129 
131  return global_file_mgr_->isBufferOnDevice(chunk_key);
132 }
133 
135  return global_file_mgr_->printSlabs();
136 }
137 
139  global_file_mgr_->clearSlabs();
140 }
141 
143  return global_file_mgr_->getMaxSize();
144 }
145 
147  return global_file_mgr_->getInUseSize();
148 }
149 
151  return global_file_mgr_->getAllocated();
152 }
153 
155  return global_file_mgr_->isAllocationCapped();
156 }
157 
159  global_file_mgr_->checkpoint();
160 }
161 
162 void PersistentStorageMgr::checkpoint(const int db_id, const int tb_id) {
163  global_file_mgr_->checkpoint(db_id, tb_id);
164 }
165 
166 AbstractBuffer* PersistentStorageMgr::alloc(const size_t num_bytes) {
167  return global_file_mgr_->alloc(num_bytes);
168 }
169 
171  global_file_mgr_->free(buffer);
172 }
173 
175  return PERSISTENT_STORAGE_MGR;
176 }
177 
179  return ToString(PERSISTENT_STORAGE_MGR);
180 }
181 
183  return global_file_mgr_->getNumChunks();
184 }
185 
187  return global_file_mgr_.get();
188 }
189 
190 void PersistentStorageMgr::removeTableRelatedDS(const int db_id, const int table_id) {
191  const ChunkKey table_key{db_id, table_id};
192  if (isChunkPrefixCacheable(table_key)) {
193  disk_cache_->clearForTablePrefix(table_key);
194  }
195  getStorageMgrForTableKey(table_key)->removeTableRelatedDS(db_id, table_id);
196 }
197 
198 bool PersistentStorageMgr::isForeignStorage(const ChunkKey& chunk_key) const {
199  CHECK(has_table_prefix(chunk_key));
200  auto db_id = chunk_key[CHUNK_KEY_DB_IDX];
201  auto table_id = chunk_key[CHUNK_KEY_TABLE_IDX];
202  auto catalog = Catalog_Namespace::Catalog::checkedGet(db_id);
203 
204  auto table = catalog->getMetadataForTableImpl(table_id, false);
205  CHECK(table);
206  return table->storageType == StorageType::FOREIGN_TABLE;
207 }
208 
210  const ChunkKey& table_key) const {
211  if (isForeignStorage(table_key)) {
212  return foreign_storage_mgr_.get();
213  } else {
214  return global_file_mgr_.get();
215  }
216 }
217 
219  return foreign_storage_mgr_.get();
220 }
221 
223  return disk_cache_ ? disk_cache_.get() : nullptr;
224 }
225 
227  CHECK(has_table_prefix(chunk_prefix));
228  // If this is an Arrow FSI table then we can't cache it.
230  chunk_prefix[CHUNK_KEY_TABLE_IDX])) {
231  return false;
232  }
234  !isForeignStorage(chunk_prefix)) ||
236 }
size_t getInUseSize() override
std::string getStringMgrType() override
std::vector< int > ChunkKey
Definition: types.h:37
size_t getAllocated() override
bool isEnabledForMutableTables() const
AbstractBuffer * alloc(const size_t num_bytes) override
PersistentStorageMgr(const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)
static Data_Namespace::AbstractBufferMgr * lookupBufferManager(const int db_id, const int table_id)
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
void deleteBuffer(const ChunkKey &chunk_key, const bool purge) override
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
foreign_storage::ForeignStorageMgr * getForeignStorageMgr() const
This file contains the class specification and related data structures for Catalog.
size_t getMaxSize() override
foreign_storage::ForeignStorageCache * getDiskCache() const
AbstractBuffer * createBuffer(const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override
bool isEnabledForFSI() const
bool isForeignStorage(const ChunkKey &chunk_key) const
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:48
An AbstractBuffer is a unit of data management for a data manager.
static std::shared_ptr< Catalog > checkedGet(const int32_t db_id)
Definition: Catalog.cpp:3846
void deleteBuffersWithPrefix(const ChunkKey &chunk_key_prefix, const bool purge) override
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
MgrType getMgrType() override
size_t getNumChunks() override
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
static PersistentStorageMgr * createPersistentStorageMgr(const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)
DiskCacheConfig disk_cache_config_
bool isEnabled() const
void copyTo(AbstractBuffer *destination_buffer, const size_t num_bytes=0)
AbstractBuffer * getBuffer(const ChunkKey &chunk_key, const size_t num_bytes) override
void removeTableRelatedDS(const int db_id, const int table_id) override
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:197
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const
AbstractBuffer * putBuffer(const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override
void fetchBuffer(const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override
bool isBufferOnDevice(const ChunkKey &chunk_key) override
static constexpr char const * FOREIGN_TABLE
std::unique_ptr< foreign_storage::ForeignStorageMgr > foreign_storage_mgr_
bool isAllocationCapped() override
std::string printSlabs() override
void free(AbstractBuffer *buffer) override