OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
GlobalFileMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
26 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
27 
28 #include <iostream>
29 #include <map>
30 #include <mutex>
31 #include <set>
32 #include "../Shared/heavyai_shared_mutex.h"
33 
34 #include "../AbstractBuffer.h"
35 #include "../AbstractBufferMgr.h"
36 #include "FileMgr.h"
37 
39 
40 using namespace Data_Namespace;
41 
42 namespace File_Namespace {
43 
44 struct FileMgrParams {
45  FileMgrParams() : epoch(-1), max_rollback_epochs(-1) {}
46  int32_t epoch;
48 };
49 
54 class GlobalFileMgr : public AbstractBufferMgr { // implements
55 
56  public:
58  GlobalFileMgr(const int32_t deviceId,
59  std::shared_ptr<ForeignStorageInterface> fsi,
60  std::string basePath = ".",
61  const size_t num_reader_threads = 0,
62  const size_t defaultPageSize = DEFAULT_PAGE_SIZE);
63 
64  ~GlobalFileMgr() override {}
65 
68  size_t pageSize = 0,
69  const size_t numBytes = 0) override {
70  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
71  }
72 
73  bool isBufferOnDevice(const ChunkKey& key) override {
74  return getFileMgr(key)->isBufferOnDevice(key);
75  }
76 
78  // Purge == true means delete the data chunks -
79  // can't undelete and revert to previous
80  // state - reclaims disk space for chunk
81  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
82  return getFileMgr(key)->deleteBuffer(key, purge);
83  }
84 
85  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
86  const bool purge = true) override;
87 
89  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
90  return getFileMgr(key)->getBuffer(key, numBytes);
91  }
92 
93  void fetchBuffer(const ChunkKey& key,
94  AbstractBuffer* destBuffer,
95  const size_t numBytes) override {
96  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
97  }
98 
106  AbstractBuffer* d,
107  const size_t numBytes = 0) override {
108  return getFileMgr(key)->putBuffer(key, d, numBytes);
109  }
110 
111  // Buffer API
112  AbstractBuffer* alloc(const size_t numBytes) override {
113  LOG(FATAL) << "Operation not supported";
114  return nullptr; // satisfy return-type warning
115  }
116 
117  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }
118 
119  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
120  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
121  inline std::string printSlabs() override { return "Not Implemented"; }
122  inline size_t getMaxSize() override { return 0; }
123  inline size_t getInUseSize() override { return 0; }
124  inline size_t getAllocated() override { return 0; }
125  inline bool isAllocationCapped() override { return false; }
126 
127  void init();
128 
130  const ChunkKey& keyPrefix) override {
131  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
132  keyPrefix);
133  }
134 
139  void checkpoint() override;
140  void checkpoint(const int32_t db_id, const int32_t tb_id) override;
141 
146  inline size_t getNumReaderThreads() { return num_reader_threads_; }
147 
148  size_t getNumChunks() override;
149 
150  void compactDataFiles(const int32_t db_id, const int32_t tb_id);
151 
152  private:
153  AbstractBufferMgr* findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id);
154  void deleteFileMgr(const int32_t db_id, const int32_t tb_id);
155 
156  public:
157  AbstractBufferMgr* findFileMgr(const int32_t db_id, const int32_t tb_id) {
159  return findFileMgrUnlocked(db_id, tb_id);
160  }
161  void setFileMgrParams(const int32_t db_id,
162  const int32_t tb_id,
163  const FileMgrParams& file_mgr_params);
164  AbstractBufferMgr* getFileMgr(const int32_t db_id, const int32_t tb_id);
165  AbstractBufferMgr* getFileMgr(const ChunkKey& key) {
166  return getFileMgr(key[0], key[1]);
167  }
168 
169  std::string getBasePath() const { return basePath_; }
170  size_t getDefaultPageSize() const { return defaultPageSize_; }
171 
172  void writeFileMgrData(FileMgr* fileMgr = 0);
173 
174  inline int32_t getDBVersion() const { return omnisci_db_version_; }
175  inline bool getDBConvert() const { return dbConvert_; }
176  inline void setDBConvert(bool val) { dbConvert_ = val; }
177 
178  void removeTableRelatedDS(const int32_t db_id, const int32_t tb_id) override;
179  void setTableEpoch(const int32_t db_id, const int32_t tb_id, const int32_t start_epoch);
180  size_t getTableEpoch(const int32_t db_id, const int32_t tb_id);
181  void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id);
182  StorageStats getStorageStats(const int32_t db_id, const int32_t tb_id);
183 
184  // For testing purposes only
185  std::shared_ptr<FileMgr> getSharedFileMgr(const int db_id, const int table_id);
186 
187  // For testing purposes only
188  void setFileMgr(const int db_id, const int table_id, std::shared_ptr<FileMgr> file_mgr);
189  void closeFileMgr(const int32_t db_id,
190  const int32_t tb_id); // A locked public wrapper for deleteFileMgr,
191  // for now for unit testing
192  protected:
193  std::shared_ptr<ForeignStorageInterface> fsi_;
194 
195  private:
196  bool existsDiffBetweenFileMgrParamsAndFileMgr(
197  FileMgr* file_mgr,
198  const FileMgrParams& file_mgr_params) const;
199  std::string basePath_;
201  int32_t
202  epoch_; /* the current epoch (time of last checkpoint) will be used for all
203  * tables except of the one for which the value of the epoch has been reset
204  * using --start-epoch option at start up to rollback this table's updates.
205  */
207  // bool isDirty_; /// true if metadata changed since last writeState()
208 
210  /* In future omnisci_db_version_ may be added to AbstractBufferMgr class.
212  * This will allow support of different dbVersions for different tables, so
213  * original tables can be generated by different versions of mapd software.
214  */
215  bool dbConvert_;
216 
218  std::map<TablePair, std::shared_ptr<FileMgr>> ownedFileMgrs_;
219  std::map<TablePair, AbstractBufferMgr*> allFileMgrs_;
220  std::map<TablePair, int32_t> max_rollback_epochs_per_table_;
221  std::map<TablePair, StorageStats> lazy_initialized_stats_;
222 
224 };
225 
226 } // namespace File_Namespace
227 
228 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
std::vector< int > ChunkKey
Definition: types.h:36
std::string getBasePath() const
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
heavyai::shared_lock< heavyai::shared_mutex > read_lock
bool isBufferOnDevice(const ChunkKey &key) override
Definition: GlobalFileMgr.h:73
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:67
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
#define LOG(tag)
Definition: Logger.h:216
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
This file includes the class specification for the FILE manager (FileMgr), and related data structure...
AbstractBufferMgr * getFileMgr(const ChunkKey &key)
std::string getStringMgrType() override
std::shared_lock< T > shared_lock
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
size_t getAllocated() override
void init(LogOptions const &log_opts)
Definition: Logger.cpp:308
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
int32_t omnisci_db_version_
default page size, used to set FileMgr defaultPageSize_
An AbstractBuffer is a unit of data management for a data manager.
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::string printSlabs() override
#define DEFAULT_PAGE_SIZE
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:81
size_t getDefaultPageSize() const
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:89
void free(AbstractBuffer *buffer) override
size_t num_reader_threads_
The OS file system path containing the files.
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
std::map< TablePair, StorageStats > lazy_initialized_stats_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:93
std::shared_timed_mutex shared_mutex
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
size_t getInUseSize() override
heavyai::shared_mutex fileMgrs_mutex_