OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PersistentStorageMgr.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "PersistentStorageMgr.h"
18 #include "Catalog/Catalog.h"
23 
25  const std::string& data_dir,
26  const size_t num_reader_threads,
27  const File_Namespace::DiskCacheConfig& disk_cache_config)
28  : AbstractBufferMgr(0), disk_cache_config_(disk_cache_config) {
29  fsi_ = std::make_shared<ForeignStorageInterface>();
32 
33  disk_cache_ =
35  ? std::make_unique<foreign_storage::ForeignStorageCache>(disk_cache_config)
36  : nullptr;
39  global_file_mgr_ = std::make_unique<File_Namespace::CachingGlobalFileMgr>(
40  0, fsi_, data_dir, num_reader_threads, disk_cache_.get());
41  } else {
42  global_file_mgr_ = std::make_unique<File_Namespace::GlobalFileMgr>(
43  0, fsi_, data_dir, num_reader_threads);
44  }
45 
49  std::make_unique<foreign_storage::CachingForeignStorageMgr>(disk_cache_.get());
50  } else {
51  foreign_storage_mgr_ = std::make_unique<foreign_storage::ForeignStorageMgr>();
52  }
53 }
54 
56  const size_t page_size,
57  const size_t initial_size) {
58  return getStorageMgrForTableKey(chunk_key)->createBuffer(
59  chunk_key, page_size, initial_size);
60 }
61 
62 void PersistentStorageMgr::deleteBuffer(const ChunkKey& chunk_key, const bool purge) {
63  getStorageMgrForTableKey(chunk_key)->deleteBuffer(chunk_key, purge);
64 }
65 
67  const bool purge) {
68  getStorageMgrForTableKey(chunk_key_prefix)
69  ->deleteBuffersWithPrefix(chunk_key_prefix, purge);
70 }
71 
73  const size_t num_bytes) {
74  return getStorageMgrForTableKey(chunk_key)->getBuffer(chunk_key, num_bytes);
75 }
76 
78  AbstractBuffer* destination_buffer,
79  const size_t num_bytes) {
80  getStorageMgrForTableKey(chunk_key)->fetchBuffer(
81  chunk_key, destination_buffer, num_bytes);
82 }
83 
85  AbstractBuffer* source_buffer,
86  const size_t num_bytes) {
87  return getStorageMgrForTableKey(chunk_key)->putBuffer(
88  chunk_key, source_buffer, num_bytes);
89 }
90 
92  ChunkMetadataVector& chunk_metadata,
93  const ChunkKey& key_prefix) {
94  getStorageMgrForTableKey(key_prefix)
95  ->getChunkMetadataVecForKeyPrefix(chunk_metadata, key_prefix);
96 }
97 
99  return global_file_mgr_->isBufferOnDevice(chunk_key);
100 }
101 
103  return global_file_mgr_->printSlabs();
104 }
105 
107  return global_file_mgr_->getMaxSize();
108 }
109 
111  return global_file_mgr_->getInUseSize();
112 }
113 
115  return global_file_mgr_->getAllocated();
116 }
117 
119  return global_file_mgr_->isAllocationCapped();
120 }
121 
123  global_file_mgr_->checkpoint();
124 }
125 
126 void PersistentStorageMgr::checkpoint(const int db_id, const int tb_id) {
127  global_file_mgr_->checkpoint(db_id, tb_id);
128 }
129 
130 AbstractBuffer* PersistentStorageMgr::alloc(const size_t num_bytes) {
131  return global_file_mgr_->alloc(num_bytes);
132 }
133 
135  global_file_mgr_->free(buffer);
136 }
137 
139  return PERSISTENT_STORAGE_MGR;
140 }
141 
143  return ToString(PERSISTENT_STORAGE_MGR);
144 }
145 
147  return global_file_mgr_->getNumChunks();
148 }
149 
151  return global_file_mgr_.get();
152 }
153 
154 void PersistentStorageMgr::removeTableRelatedDS(const int db_id, const int table_id) {
155  getStorageMgrForTableKey({db_id, table_id})->removeTableRelatedDS(db_id, table_id);
156 }
157 
158 bool PersistentStorageMgr::isForeignStorage(const ChunkKey& chunk_key) const {
159  CHECK(has_table_prefix(chunk_key));
160  auto db_id = chunk_key[CHUNK_KEY_DB_IDX];
161  auto table_id = chunk_key[CHUNK_KEY_TABLE_IDX];
162  auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
163 
164  // if catalog doesnt exist at this point we must be in an old migration.
165  // Old migration can not, at this point 5.5.1, be using foreign storage
166  // so this hack is to avoid the crash, when migrating old
167  // catalogs that have not been upgraded over time due to issue
168  // [BE-5728]
169  if (!catalog) {
170  return false;
171  }
172 
173  auto table = catalog->getMetadataForTable(table_id, false);
174  CHECK(table);
175  return table->storageType == StorageType::FOREIGN_TABLE;
176 }
177 
179  const ChunkKey& table_key) const {
180  if (isForeignStorage(table_key)) {
181  return foreign_storage_mgr_.get();
182  } else {
183  return global_file_mgr_.get();
184  }
185 }
186 
188  return foreign_storage_mgr_.get();
189 }
190 
192  return disk_cache_ ? disk_cache_.get() : nullptr;
193 }
size_t getInUseSize() override
std::string getStringMgrType() override
std::vector< int > ChunkKey
Definition: types.h:37
size_t getAllocated() override
PersistentStorageMgr(const std::string &data_dir, const size_t num_reader_threads, const File_Namespace::DiskCacheConfig &disk_cache_config)
AbstractBuffer * alloc(const size_t num_bytes) override
void registerArrowForeignStorage(std::shared_ptr< ForeignStorageInterface > fsi)
#define CHUNK_KEY_DB_IDX
Definition: types.h:39
void deleteBuffer(const ChunkKey &chunk_key, const bool purge) override
void registerArrowCsvForeignStorage(std::shared_ptr< ForeignStorageInterface > fsi)
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
foreign_storage::ForeignStorageMgr * getForeignStorageMgr() const
This file contains the class specification and related data structures for Catalog.
size_t getMaxSize() override
foreign_storage::ForeignStorageCache * getDiskCache() const
static SysCatalog & instance()
Definition: SysCatalog.h:325
AbstractBuffer * createBuffer(const ChunkKey &chunk_key, const size_t page_size, const size_t initial_size) override
std::shared_ptr< ForeignStorageInterface > fsi_
bool isForeignStorage(const ChunkKey &chunk_key) const
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunk_metadata, const ChunkKey &chunk_key_prefix) override
bool has_table_prefix(const ChunkKey &key)
Definition: types.h:49
An AbstractBuffer is a unit of data management for a data manager.
void deleteBuffersWithPrefix(const ChunkKey &chunk_key_prefix, const bool purge) override
MgrType getMgrType() override
std::shared_ptr< Catalog > getCatalog(const std::string &dbName)
size_t getNumChunks() override
std::unique_ptr< foreign_storage::ForeignStorageCache > disk_cache_
File_Namespace::DiskCacheConfig disk_cache_config_
AbstractBuffer * getBuffer(const ChunkKey &chunk_key, const size_t num_bytes) override
void removeTableRelatedDS(const int db_id, const int table_id) override
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
#define CHECK(condition)
Definition: Logger.h:209
AbstractBufferMgr * getStorageMgrForTableKey(const ChunkKey &table_key) const
AbstractBuffer * putBuffer(const ChunkKey &chunk_key, AbstractBuffer *source_buffer, const size_t num_bytes) override
void fetchBuffer(const ChunkKey &chunk_key, AbstractBuffer *destination_buffer, const size_t num_bytes) override
bool isBufferOnDevice(const ChunkKey &chunk_key) override
static constexpr char const * FOREIGN_TABLE
std::unique_ptr< foreign_storage::ForeignStorageMgr > foreign_storage_mgr_
bool isAllocationCapped() override
std::string printSlabs() override
void free(AbstractBuffer *buffer) override