OmniSciDB  8fa3bf436f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
GlobalFileMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
27 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
28 
29 #include <iostream>
30 #include <map>
31 #include <mutex>
32 #include <set>
33 #include "../Shared/mapd_shared_mutex.h"
34 
35 #include "../AbstractBuffer.h"
36 #include "../AbstractBufferMgr.h"
37 #include "FileMgr.h"
38 
40 
41 using namespace Data_Namespace;
42 
43 namespace File_Namespace {
44 
45 struct FileMgrParams {
46  FileMgrParams() : epoch(-1), max_rollback_epochs(-1) {}
47  int32_t epoch;
49 };
50 
55 class GlobalFileMgr : public AbstractBufferMgr { // implements
56 
57  public:
59  GlobalFileMgr(const int32_t deviceId,
60  std::shared_ptr<ForeignStorageInterface> fsi,
61  std::string basePath = ".",
62  const size_t num_reader_threads = 0,
63  const size_t defaultPageSize = DEFAULT_PAGE_SIZE);
64 
67  size_t pageSize = 0,
68  const size_t numBytes = 0) override {
69  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
70  }
71 
72  bool isBufferOnDevice(const ChunkKey& key) override {
73  return getFileMgr(key)->isBufferOnDevice(key);
74  }
75 
77  // Purge == true means delete the data chunks -
78  // can't undelete and revert to previous
79  // state - reclaims disk space for chunk
80  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
81  return getFileMgr(key)->deleteBuffer(key, purge);
82  }
83 
84  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
85  const bool purge = true) override;
86 
88  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
89  return getFileMgr(key)->getBuffer(key, numBytes);
90  }
91 
92  void fetchBuffer(const ChunkKey& key,
93  AbstractBuffer* destBuffer,
94  const size_t numBytes) override {
95  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
96  }
97 
105  AbstractBuffer* d,
106  const size_t numBytes = 0) override {
107  return getFileMgr(key)->putBuffer(key, d, numBytes);
108  }
109 
110  // Buffer API
111  AbstractBuffer* alloc(const size_t numBytes) override {
112  LOG(FATAL) << "Operation not supported";
113  return nullptr; // satisfy return-type warning
114  }
115 
116  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }
117 
118  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
119  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
120  inline std::string printSlabs() override { return "Not Implemented"; }
121  inline void clearSlabs() override { /* noop */
122  }
123  inline size_t getMaxSize() override { return 0; }
124  inline size_t getInUseSize() override { return 0; }
125  inline size_t getAllocated() override { return 0; }
126  inline bool isAllocationCapped() override { return false; }
127 
128  void init();
129 
131  const ChunkKey& keyPrefix) override {
132  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
133  keyPrefix);
134  }
135 
140  void checkpoint() override;
141  void checkpoint(const int32_t db_id, const int32_t tb_id) override;
142 
147  inline size_t getNumReaderThreads() { return num_reader_threads_; }
148 
149  size_t getNumChunks() override;
150 
151  void compactDataFiles(const int32_t db_id, const int32_t tb_id);
152 
153  private:
154  AbstractBufferMgr* findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id);
155  void deleteFileMgr(const int32_t db_id, const int32_t tb_id);
156 
157  public:
158  AbstractBufferMgr* findFileMgr(const int32_t db_id, const int32_t tb_id) {
159  mapd_shared_lock<mapd_shared_mutex> read_lock(fileMgrs_mutex_);
160  return findFileMgrUnlocked(db_id, tb_id);
161  }
162  void setFileMgrParams(const int32_t db_id,
163  const int32_t tb_id,
164  const FileMgrParams& file_mgr_params);
165  AbstractBufferMgr* getFileMgr(const int32_t db_id, const int32_t tb_id);
166  AbstractBufferMgr* getFileMgr(const ChunkKey& key) {
167  return getFileMgr(key[0], key[1]);
168  }
169 
170  std::string getBasePath() const { return basePath_; }
171  size_t getDefaultPageSize() const { return defaultPageSize_; }
172 
173  void writeFileMgrData(FileMgr* fileMgr = 0);
174 
175  inline int32_t getDBVersion() const { return omnisci_db_version_; }
176  inline bool getDBConvert() const { return dbConvert_; }
177  inline void setDBConvert(bool val) { dbConvert_ = val; }
178 
179  void removeTableRelatedDS(const int32_t db_id, const int32_t tb_id) override;
180  void setTableEpoch(const int32_t db_id, const int32_t tb_id, const int32_t start_epoch);
181  size_t getTableEpoch(const int32_t db_id, const int32_t tb_id);
182  StorageStats getStorageStats(const int32_t db_id, const int32_t tb_id);
183 
184  // For testing purposes only
185  std::shared_ptr<FileMgr> getSharedFileMgr(const int db_id, const int table_id);
186 
187  // For testing purposes only
188  void setFileMgr(const int db_id, const int table_id, std::shared_ptr<FileMgr> file_mgr);
189  void closeFileMgr(const int32_t db_id,
190  const int32_t tb_id); // A locked public wrapper for deleteFileMgr,
191  // for now for unit testing
192 
193  private:
194  bool existsDiffBetweenFileMgrParamsAndFileMgr(
195  FileMgr* file_mgr,
196  const FileMgrParams& file_mgr_params) const;
197  std::string basePath_;
199  int32_t
200  epoch_; /* the current epoch (time of last checkpoint) will be used for all
201  * tables except of the one for which the value of the epoch has been reset
202  * using --start-epoch option at start up to rollback this table's updates.
203  */
205  // bool isDirty_; /// true if metadata changed since last writeState()
206 
208  /* In future omnisci_db_version_ may be added to AbstractBufferMgr class.
210  * This will allow support of different dbVersions for different tables, so
211  * original tables can be generated by different versions of mapd software.
212  */
213  bool dbConvert_;
214 
216  std::map<TablePair, std::shared_ptr<FileMgr>> ownedFileMgrs_;
217  std::map<TablePair, AbstractBufferMgr*> allFileMgrs_;
218  std::map<TablePair, int32_t> max_rollback_epochs_per_table_;
219  std::shared_ptr<ForeignStorageInterface> fsi_;
220 
222 };
223 
224 } // namespace File_Namespace
225 
226 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
std::vector< int > ChunkKey
Definition: types.h:37
std::string getBasePath() const
int32_t epoch_
number of threads used when loading data
std::shared_ptr< ForeignStorageInterface > fsi_
tuple d
Definition: test_fsi.py:9
bool isBufferOnDevice(const ChunkKey &key) override
Definition: GlobalFileMgr.h:72
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:66
std::map< TablePair, std::shared_ptr< FileMgr > > ownedFileMgrs_
#define LOG(tag)
Definition: Logger.h:194
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
AbstractBufferMgr * getFileMgr(const ChunkKey &key)
std::string getStringMgrType() override
std::map< TablePair, AbstractBufferMgr * > allFileMgrs_
size_t getAllocated() override
void init(LogOptions const &log_opts)
Definition: Logger.cpp:280
std::shared_timed_mutex mapd_shared_mutex
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
int32_t omnisci_db_version_
default page size, used to set FileMgr defaultPageSize_
An AbstractBuffer is a unit of data management for a data manager.
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::string printSlabs() override
#define DEFAULT_PAGE_SIZE
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:80
size_t getDefaultPageSize() const
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:88
void free(AbstractBuffer *buffer) override
mapd_shared_lock< mapd_shared_mutex > read_lock
size_t num_reader_threads_
The OS file system path containing the files.
std::map< TablePair, int32_t > max_rollback_epochs_per_table_
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
mapd_shared_mutex fileMgrs_mutex_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:92
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
size_t getInUseSize() override