OmniSciDB  94e8789169
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
GlobalFileMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
27 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
28 
29 #include <iostream>
30 #include <map>
31 #include <mutex>
32 #include <set>
33 #include "../Shared/mapd_shared_mutex.h"
34 
35 #include "../AbstractBuffer.h"
36 #include "../AbstractBufferMgr.h"
37 #include "FileMgr.h"
38 
39 using namespace Data_Namespace;
40 
41 namespace File_Namespace {
42 
43 struct FileMgrParams {
44  FileMgrParams() : epoch(-1), max_rollback_epochs(-1) {}
45  int32_t epoch;
47 };
48 
53 class GlobalFileMgr : public AbstractBufferMgr { // implements
54 
55  public:
57  GlobalFileMgr(const int32_t deviceId,
58  std::string basePath = ".",
59  const size_t num_reader_threads = 0,
60  const size_t defaultPageSize = DEFAULT_PAGE_SIZE);
61 
64  size_t pageSize = 0,
65  const size_t numBytes = 0) override {
66  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
67  }
68 
69  bool isBufferOnDevice(const ChunkKey& key) override {
70  return getFileMgr(key)->isBufferOnDevice(key);
71  }
72 
74  // Purge == true means delete the data chunks -
75  // can't undelete and revert to previous
76  // state - reclaims disk space for chunk
77  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
78  return getFileMgr(key)->deleteBuffer(key, purge);
79  }
80 
81  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
82  const bool purge = true) override;
83 
85  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
86  return getFileMgr(key)->getBuffer(key, numBytes);
87  }
88 
89  void fetchBuffer(const ChunkKey& key,
90  AbstractBuffer* destBuffer,
91  const size_t numBytes) override {
92  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
93  }
94 
102  AbstractBuffer* d,
103  const size_t numBytes = 0) override {
104  return getFileMgr(key)->putBuffer(key, d, numBytes);
105  }
106 
107  // Buffer API
108  AbstractBuffer* alloc(const size_t numBytes) override {
109  LOG(FATAL) << "Operation not supported";
110  return nullptr; // satisfy return-type warning
111  }
112 
113  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }
114 
115  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
116  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
117  inline std::string printSlabs() override { return "Not Implemented"; }
118  inline void clearSlabs() override { /* noop */
119  }
120  inline size_t getMaxSize() override { return 0; }
121  inline size_t getInUseSize() override { return 0; }
122  inline size_t getAllocated() override { return 0; }
123  inline bool isAllocationCapped() override { return false; }
124 
125  void init();
126 
128  const ChunkKey& keyPrefix) override {
129  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
130  keyPrefix);
131  }
132 
137  void checkpoint() override;
138  void checkpoint(const int32_t db_id, const int32_t tb_id) override;
139 
144  inline size_t getNumReaderThreads() { return num_reader_threads_; }
145 
146  size_t getNumChunks() override;
147 
148  private:
149  AbstractBufferMgr* findFileMgrUnlocked(const int32_t db_id, const int32_t tb_id);
150  void deleteFileMgr(const int32_t db_id, const int32_t tb_id);
151 
152  public:
153  AbstractBufferMgr* findFileMgr(const int32_t db_id, const int32_t tb_id) {
154  mapd_shared_lock<mapd_shared_mutex> read_lock(fileMgrs_mutex_);
155  return findFileMgrUnlocked(db_id, tb_id);
156  }
157  void setFileMgrParams(const int32_t db_id,
158  const int32_t tb_id,
159  const FileMgrParams& file_mgr_params);
160  AbstractBufferMgr* getFileMgr(const int32_t db_id, const int32_t tb_id);
161  AbstractBufferMgr* getFileMgr(const ChunkKey& key) {
162  return getFileMgr(key[0], key[1]);
163  }
164 
165  std::string getBasePath() const { return basePath_; }
166  size_t getDefaultPageSize() const { return defaultPageSize_; }
167 
168  void writeFileMgrData(FileMgr* fileMgr = 0);
169 
170  inline int32_t getDBVersion() const { return omnisci_db_version_; }
171  inline bool getDBConvert() const { return dbConvert_; }
172  inline void setDBConvert(bool val) { dbConvert_ = val; }
173 
174  void removeTableRelatedDS(const int32_t db_id, const int32_t tb_id) override;
175  void setTableEpoch(const int32_t db_id, const int32_t tb_id, const int32_t start_epoch);
176  size_t getTableEpoch(const int32_t db_id, const int32_t tb_id);
177  StorageStats getStorageStats(const int32_t db_id, const int32_t tb_id);
178 
179  // For testing purposes only
180  std::shared_ptr<FileMgr> getSharedFileMgr(const int db_id, const int table_id);
181 
182  // For testing purposes only
183  void setFileMgr(const int db_id, const int table_id, std::shared_ptr<FileMgr> file_mgr);
184  void closeFileMgr(const int32_t db_id,
185  const int32_t tb_id); // A locked public wrapper for deleteFileMgr,
186  // for now for unit testing
187 
188  private:
189  bool existsDiffBetweenFileMgrParamsAndFileMgr(
190  FileMgr* file_mgr,
191  const FileMgrParams& file_mgr_params) const;
192  std::string basePath_;
194  int32_t
195  epoch_; /* the current epoch (time of last checkpoint) will be used for all
196  * tables except of the one for which the value of the epoch has been reset
197  * using --start-epoch option at start up to rollback this table's updates.
198  */
200  // bool isDirty_; /// true if metadata changed since last writeState()
201 
203  /* In future omnisci_db_version_ may be added to AbstractBufferMgr class.
205  * This will allow support of different dbVersions for different tables, so
206  * original tables can be generated by different versions of mapd software.
207  */
208  bool dbConvert_;
209 
211  std::map<std::pair<int32_t, int32_t>, std::shared_ptr<FileMgr>> ownedFileMgrs_;
212  std::map<std::pair<int32_t, int32_t>, AbstractBufferMgr*> allFileMgrs_;
213 
215 };
216 
217 } // namespace File_Namespace
218 
219 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
std::vector< int > ChunkKey
Definition: types.h:37
std::string getBasePath() const
std::map< std::pair< int32_t, int32_t >, AbstractBufferMgr * > allFileMgrs_
int32_t epoch_
number of threads used when loading data
bool isBufferOnDevice(const ChunkKey &key) override
Definition: GlobalFileMgr.h:69
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:63
#define LOG(tag)
Definition: Logger.h:188
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
AbstractBufferMgr * getFileMgr(const ChunkKey &key)
std::string getStringMgrType() override
std::map< std::pair< int32_t, int32_t >, std::shared_ptr< FileMgr > > ownedFileMgrs_
#define DEFAULT_PAGE_SIZE
Definition: FileMgr.h:47
size_t getAllocated() override
void init(LogOptions const &log_opts)
Definition: Logger.cpp:280
std::shared_timed_mutex mapd_shared_mutex
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
int32_t omnisci_db_version_
default page size, used to set FileMgr defaultPageSize_
An AbstractBuffer is a unit of data management for a data manager.
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::string printSlabs() override
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:77
size_t getDefaultPageSize() const
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:85
void free(AbstractBuffer *buffer) override
mapd_shared_lock< mapd_shared_mutex > read_lock
size_t num_reader_threads_
The OS file system path containing the files.
AbstractBufferMgr * findFileMgr(const int32_t db_id, const int32_t tb_id)
mapd_shared_mutex fileMgrs_mutex_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:89
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
size_t getInUseSize() override