OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
GlobalFileMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
27 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
28 
29 #include <iostream>
30 #include <map>
31 #include <mutex>
32 #include <set>
33 #include "../Shared/mapd_shared_mutex.h"
34 
35 #include "../AbstractBuffer.h"
36 #include "../AbstractBufferMgr.h"
37 #include "FileMgr.h"
38 
40 
41 using namespace Data_Namespace;
42 
43 namespace File_Namespace {
44 
45 struct FileMgrParams {
46  FileMgrParams() : epoch(-1), max_rollback_epochs(-1) {}
47  int32_t epoch;
49 };
50 
55 class GlobalFileMgr : public AbstractBufferMgr { // implements
56 
57  public:
59  GlobalFileMgr(const int32_t deviceId,
60  std::shared_ptr<ForeignStorageInterface> fsi,
61  std::string basePath = ".",
62  const size_t num_reader_threads = 0,
63  const size_t defaultPageSize = DEFAULT_PAGE_SIZE);
64 
65  ~GlobalFileMgr() override {}
66 
69  size_t pageSize = 0,
70  const size_t numBytes = 0) override {
71  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
72  }
73 
74  bool isBufferOnDevice(const ChunkKey& key) override {
75  return getFileMgr(key)->isBufferOnDevice(key);
76  }
77 
79  // Purge == true means delete the data chunks -
80  // can't undelete and revert to previous
81  // state - reclaims disk space for chunk
82  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
83  return getFileMgr(key)->deleteBuffer(key, purge);
84  }
85 
86  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
87  const bool purge = true) override;
88 
90  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
91  return getFileMgr(key)->getBuffer(key, numBytes);
92  }
93 
94  void fetchBuffer(const ChunkKey& key,
95  AbstractBuffer* destBuffer,
96  const size_t numBytes) override {
97  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
98  }
99 
107  AbstractBuffer* d,
108  const size_t numBytes = 0) override {
109  return getFileMgr(key)->putBuffer(key, d, numBytes);
110  }
111 
112  // Buffer API
113  AbstractBuffer* alloc(const size_t numBytes) override {
114  LOG(FATAL) << "Operation not supported";
115  return nullptr; // satisfy return-type warning
116  }
117 
118  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }
119 
120  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
121  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
122  inline std::string printSlabs() override { return "Not Implemented"; }
123  inline size_t getMaxSize() override { return 0; }
124  inline size_t getInUseSize() override { return 0; }
125  inline size_t getAllocated() override { return 0; }
126  inline bool isAllocationCapped() override { return false; }
127 
128  void init();
129 
131  const ChunkKey& keyPrefix) override {
132  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
133  keyPrefix);
134  }
135 
140  void checkpoint() override;
141  void checkpoint(const int32_t db_id, const int32_t tb_id) override;
142 
147  inline size_t getNumReaderThreads() { return num_reader_threads_; }
148 
149  size_t getNumChunks() override;
150 
151  void compactDataFiles(const int32_t db_id, const int32_t tb_id);
152 
153  private:
154  AbstractBufferMgr* findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id);
155  void deleteFileMgr(const int32_t db_id, const int32_t tb_id);
156 
157  public:
158  AbstractBufferMgr* findFileMgr(const int32_t db_id, const int32_t tb_id) {
159  mapd_shared_lock<mapd_shared_mutex> read_lock(fileMgrs_mutex_);
160  return findFileMgrUnlocked(db_id, tb_id);
161  }
162  void setFileMgrParams(const int32_t db_id,
163  const int32_t tb_id,
164  const FileMgrParams& file_mgr_params);
165  AbstractBufferMgr* getFileMgr(const int32_t db_id, const int32_t tb_id);
166  AbstractBufferMgr* getFileMgr(const ChunkKey& key) {
167  return getFileMgr(key[0], key[1]);
168  }
169 
170  std::string getBasePath() const { return basePath_; }
171  size_t getDefaultPageSize() const { return defaultPageSize_; }
172 
173  void writeFileMgrData(FileMgr* fileMgr = 0);
174 
175  inline int32_t getDBVersion() const { return omnisci_db_version_; }
176  inline bool getDBConvert() const { return dbConvert_; }
177  inline void setDBConvert(bool val) { dbConvert_ = val; }
178 
179  void removeTableRelatedDS(const int32_t db_id, const int32_t tb_id) override;
180  void setTableEpoch(const int32_t db_id, const int32_t tb_id, const int32_t start_epoch);
181  size_t getTableEpoch(const int32_t db_id, const int32_t tb_id);
182  StorageStats getStorageStats(const int32_t db_id, const int32_t tb_id);
183 
184  // For testing purposes only
185  std::shared_ptr<FileMgr> getSharedFileMgr(const int db_id, const int table_id);
186 
187  // For testing purposes only
188  void setFileMgr(const int db_id, const int table_id, std::shared_ptr<FileMgr> file_mgr);
189  void closeFileMgr(const int32_t db_id,
190  const int32_t tb_id); // A locked public wrapper for deleteFileMgr,
191  // for now for unit testing
192  protected:
193  std::shared_ptr<ForeignStorageInterface> fsi_;
194 
195  private:
196  bool existsDiffBetweenFileMgrParamsAndFileMgr(
197  FileMgr* file_mgr,
198  const FileMgrParams& file_mgr_params) const;
199  std::string basePath_;
201  int32_t
202  epoch_; /* the current epoch (time of last checkpoint) will be used for all
203  * tables except of the one for which the value of the epoch has been reset
204  * using --start-epoch option at start up to rollback this table's updates.
205  */
207  // bool isDirty_; /// true if metadata changed since last writeState()
208 
210  /* In future omnisci_db_version_ may be added to AbstractBufferMgr class.
212  * This will allow support of different dbVersions for different tables, so
213  * original tables can be generated by different versions of mapd software.
214  */
215  bool dbConvert_;
216 
218  std::map<TablePair, std::shared_ptr<FileMgr>> ownedFileMgrs_;
219  std::map<TablePair, AbstractBufferMgr*> allFileMgrs_;
220  std::map<TablePair, int32_t> max_rollback_epochs_per_table_;
221 
223 };
224 
225 } // namespace File_Namespace
226 
227 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
std::vector< int > ChunkKey
Definition: types.h:37
std::string getBasePath() const
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
bool isBufferOnDevice(const ChunkKey &key) override
Definition: GlobalFileMgr.h:74
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:68
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
#define LOG(tag)
Definition: Logger.h:203
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
AbstractBufferMgr * getFileMgr(const ChunkKey &key)
std::string getStringMgrType() override
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
size_t getAllocated() override
void init(LogOptions const &log_opts)
Definition: Logger.cpp:290
std::shared_timed_mutex mapd_shared_mutex
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
int32_t omnisci_db_version_
default page size, used to set FileMgr defaultPageSize_
An AbstractBuffer is a unit of data management for a data manager.
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::string printSlabs() override
#define DEFAULT_PAGE_SIZE
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:82
size_t getDefaultPageSize() const
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:90
void free(AbstractBuffer *buffer) override
mapd_shared_lock< mapd_shared_mutex > read_lock
size_t num_reader_threads_
The OS file system path containing the files.
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
mapd_shared_mutex fileMgrs_mutex_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:94
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
size_t getInUseSize() override