OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
PersistentStorageMgr.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "PersistentStorageMgr.h"
18 #include "Catalog/Catalog.h"
23 
25  const std::string& data_dir,
26  const size_t num_reader_threads,
27  const DiskCacheConfig& config) {
28  if (config.isEnabledForMutableTables()) {
29  return new MutableCachePersistentStorageMgr(data_dir, num_reader_threads, config);
30  } else {
31  return new PersistentStorageMgr(data_dir, num_reader_threads, config);
32  }
33 }
34 
35 PersistentStorageMgr::PersistentStorageMgr(const std::string& data_dir,
36  const size_t num_reader_threads,
37  const DiskCacheConfig& disk_cache_config)
38  : AbstractBufferMgr(0), disk_cache_config_(disk_cache_config) {
39  fsi_ = std::make_shared<ForeignStorageInterface>();
42 
43  global_file_mgr_ = std::make_unique<File_Namespace::GlobalFileMgr>(
44  0, fsi_, data_dir, num_reader_threads);
45  disk_cache_ =
47  ? std::make_unique<foreign_storage::ForeignStorageCache>(disk_cache_config)
48  : nullptr;
51  ? std::make_unique<foreign_storage::CachingForeignStorageMgr>(disk_cache_.get())
52  : std::make_unique<foreign_storage::ForeignStorageMgr>();
53 }
54 
56  const size_t page_size,
57  const size_t initial_size) {
58  return getStorageMgrForTableKey(chunk_key)->createBuffer(
59  chunk_key, page_size, initial_size);
60 }
61 
62 void PersistentStorageMgr::deleteBuffer(const ChunkKey& chunk_key, const bool purge) {
63  getStorageMgrForTableKey(chunk_key)->deleteBuffer(chunk_key, purge);
64 }
65 
67  const bool purge) {
68  getStorageMgrForTableKey(chunk_key_prefix)
69  ->deleteBuffersWithPrefix(chunk_key_prefix, purge);
70 }
71 
73  const size_t num_bytes) {
74  return getStorageMgrForTableKey(chunk_key)->getBuffer(chunk_key, num_bytes);
75 }
76 
78  AbstractBuffer* destination_buffer,
79  const size_t num_bytes) {
80  AbstractBufferMgr* mgr = getStorageMgrForTableKey(chunk_key);
81  if (isChunkPrefixCacheable(chunk_key)) {
82  AbstractBuffer* buffer = disk_cache_->getCachedChunkIfExists(chunk_key);
83  if (buffer) {
84  buffer->copyTo(destination_buffer, num_bytes);
85  return;
86  } else {
87  mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
88  if (!isForeignStorage(chunk_key)) {
89  // Foreign storage will read into cache buffers directly if enabled, so we do
90  // not want to cache foreign table chunks here as they will already be cached.
91  disk_cache_->cacheChunk(chunk_key, destination_buffer);
92  }
93  return;
94  }
95  }
96  mgr->fetchBuffer(chunk_key, destination_buffer, num_bytes);
97 }
98 
100  AbstractBuffer* source_buffer,
101  const size_t num_bytes) {
102  return getStorageMgrForTableKey(chunk_key)->putBuffer(
103  chunk_key, source_buffer, num_bytes);
104 }
105 
107  ChunkMetadataVector& chunk_metadata,
108  const ChunkKey& keyPrefix) {
109  CHECK(has_table_prefix(keyPrefix));
110  // If the disk has any cached metadata for a prefix then it is guaranteed to have all
111  // metadata for that table, so we can return a complete set. If it has no metadata,
112  // then it may be that the table has no data, or that it's just not cached, so we need
113  // to go to storage to check.
114  if (isChunkPrefixCacheable(keyPrefix)) {
115  if (disk_cache_->hasCachedMetadataForKeyPrefix(keyPrefix)) {
116  disk_cache_->getCachedMetadataVecForKeyPrefix(chunk_metadata, keyPrefix);
117  return;
118  } else { // if we have no cached data attempt a recovery.
119  if (disk_cache_->recoverCacheForTable(chunk_metadata, get_table_key(keyPrefix))) {
120  return;
121  }
122  }
123  getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
124  keyPrefix);
125  disk_cache_->cacheMetadataVec(chunk_metadata);
126  } else {
127  getStorageMgrForTableKey(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunk_metadata,
128  keyPrefix);
129  }
130 }
131 
133  return global_file_mgr_->isBufferOnDevice(chunk_key);
134 }
135 
137  return global_file_mgr_->printSlabs();
138 }
139 
141  global_file_mgr_->clearSlabs();
142 }
143 
145  return global_file_mgr_->getMaxSize();
146 }
147 
149  return global_file_mgr_->getInUseSize();
150 }
151 
153  return global_file_mgr_->getAllocated();
154 }
155 
157  return global_file_mgr_->isAllocationCapped();
158 }
159 
161  global_file_mgr_->checkpoint();
162 }
163 
164 void PersistentStorageMgr::checkpoint(const int db_id, const int tb_id) {
165  global_file_mgr_->checkpoint(db_id, tb_id);
166 }
167 
168 AbstractBuffer* PersistentStorageMgr::alloc(const size_t num_bytes) {
169  return global_file_mgr_->alloc(num_bytes);
170 }
171 
173  global_file_mgr_->free(buffer);
174 }
175 
177  return PERSISTENT_STORAGE_MGR;
178 }
179 
181  return ToString(PERSISTENT_STORAGE_MGR);
182 }
183 
185  return global_file_mgr_->getNumChunks();
186 }
187 
189  return global_file_mgr_.get();
190 }
191 
192 void PersistentStorageMgr::removeTableRelatedDS(const int db_id, const int table_id) {
193  const ChunkKey table_key{db_id, table_id};
194  if (isChunkPrefixCacheable(table_key)) {
195  disk_cache_->clearForTablePrefix(table_key);
196  }
197  getStorageMgrForTableKey(table_key)->removeTableRelatedDS(db_id, table_id);
198 }
199 
200 bool PersistentStorageMgr::isForeignStorage(const ChunkKey& chunk_key) const {
201  CHECK(has_table_prefix(chunk_key));
202  auto db_id = chunk_key[CHUNK_KEY_DB_IDX];
203  auto table_id = chunk_key[CHUNK_KEY_TABLE_IDX];
204  auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
205 
206  // if catalog doesnt exist at this point we must be in an old migration.
207  // Old migration can not, at this point 5.5.1, be using foreign storage
208  // so this hack is to avoid the crash, when migrating old
209  // catalogs that have not been upgraded over time due to issue
210  // [BE-5728]
211  if (!catalog) {
212  return false;
213  }
214 
215  auto table = catalog->getMetadataForTableImpl(table_id, false);
216  CHECK(table);
217  return table->storageType == StorageType::FOREIGN_TABLE;
218 }
219 
221  const ChunkKey& table_key) const {
222  if (isForeignStorage(table_key)) {
223  return foreign_storage_mgr_.get();
224  } else {
225  return global_file_mgr_.get();
226  }
227 }
228 
230  return foreign_storage_mgr_.get();
231 }
232 
234  return disk_cache_ ? disk_cache_.get() : nullptr;
235 }
236 
238  CHECK(has_table_prefix(chunk_prefix));
239  // If this is an Arrow FSI table then we can't cache it.
240  if (fsi_->lookupBufferManager(chunk_prefix[CHUNK_KEY_DB_IDX],
241  chunk_prefix[CHUNK_KEY_TABLE_IDX])) {
242  return false;
243  }
245  !isForeignStorage(chunk_prefix)) ||
247 }
size_t getInUseSize() override
std::string getStringMgrType() override
std::vector< int > ChunkKey
Definition: types.h:37
size_t getAllocated() override
bool isEnabledForMutableTables() const
AbstractBuffer * alloc(const size_t num_bytes) override
void registerArrowForeignStorage(std::shared_ptr< ForeignStorageInterface > fsi)
PersistentStorageMgr(const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
void deleteBuffer(const ChunkKey &chunk_key, const bool purge) override
void registerArrowCsvForeignStorage(std::shared_ptr< ForeignStorageInterface > fsi)
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
ChunkKey get_table_key(const ChunkKey &key)
Definition: types.h:52
foreign_storage::ForeignStorageMgr * getForeignStorageMgr() const
This file contains the class specification and related data structures for Catalog.
size_t getMaxSize() override
foreign_storage::ForeignStorageCache * getDiskCache() const
static SysCatalog & instance()
Definition: SysCatalog.h:292
AbstractBuffer * createBuffer(const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override
std::shared_ptr< ForeignStorageInterface > fsi_
bool isEnabledForFSI() const
bool isForeignStorage(const ChunkKey &chunk_key) const
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:48
An AbstractBuffer is a unit of data management for a data manager.
void deleteBuffersWithPrefix(const ChunkKey &chunk_key_prefix, const bool purge) override
bool isChunkPrefixCacheable(const ChunkKey &chunk_prefix) const
MgrType getMgrType() override
std::shared_ptr< Catalog > getCatalog(const std::string &dbName)
size_t getNumChunks() override
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
static PersistentStorageMgr * createPersistentStorageMgr(const std::string &data_dir, const size_t num_reader_threads, const DiskCacheConfig &disk_cache_config)
DiskCacheConfig disk_cache_config_
bool isEnabled() const
void copyTo(AbstractBuffer *destination_buffer, const size_t num_bytes=0)
AbstractBuffer * getBuffer(const ChunkKey &chunk_key, const size_t num_bytes) override
void removeTableRelatedDS(const int db_id, const int table_id) override
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:203
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const
AbstractBuffer * putBuffer(const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override
void fetchBuffer(const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override
bool isBufferOnDevice(const ChunkKey &chunk_key) override
static constexpr char const * FOREIGN_TABLE
std::unique_ptr< foreign_storage::ForeignStorageMgr > foreign_storage_mgr_
bool isAllocationCapped() override
std::string printSlabs() override
void free(AbstractBuffer *buffer) override