OmniSciDB  0b528656ed
GlobalFileMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
27 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
28 
29 #include <iostream>
30 #include <map>
31 #include <mutex>
32 #include <set>
33 #include "../Shared/mapd_shared_mutex.h"
34 
35 #include "../AbstractBuffer.h"
36 #include "../AbstractBufferMgr.h"
37 #include "FileMgr.h"
38 
39 using namespace Data_Namespace;
40 
41 namespace File_Namespace {
42 
47 class GlobalFileMgr : public AbstractBufferMgr { // implements
48 
49  public:
51  GlobalFileMgr(const int deviceId,
52  std::string basePath = ".",
53  const size_t num_reader_threads = 0,
54  const size_t defaultPageSize = 2097152);
55 
58  size_t pageSize = 0,
59  const size_t numBytes = 0) override {
60  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
61  }
62 
63  bool isBufferOnDevice(const ChunkKey& key) override {
64  return getFileMgr(key)->isBufferOnDevice(key);
65  }
66 
68  // Purge == true means delete the data chunks -
69  // can't undelete and revert to previous
70  // state - reclaims disk space for chunk
71  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
72  return getFileMgr(key)->deleteBuffer(key, purge);
73  }
74 
75  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
76  const bool purge = true) override;
77 
79  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
80  return getFileMgr(key)->getBuffer(key, numBytes);
81  }
82 
83  void fetchBuffer(const ChunkKey& key,
84  AbstractBuffer* destBuffer,
85  const size_t numBytes) override {
86  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
87  }
88 
96  AbstractBuffer* d,
97  const size_t numBytes = 0) override {
98  return getFileMgr(key)->putBuffer(key, d, numBytes);
99  }
100 
101  // Buffer API
102  AbstractBuffer* alloc(const size_t numBytes) override {
103  LOG(FATAL) << "Operation not supported";
104  return nullptr; // satisfy return-type warning
105  }
106 
107  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }
108 
109  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
110  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
111  inline std::string printSlabs() override { return "Not Implemented"; }
112  inline void clearSlabs() override { /* noop */
113  }
114  inline size_t getMaxSize() override { return 0; }
115  inline size_t getInUseSize() override { return 0; }
116  inline size_t getAllocated() override { return 0; }
117  inline bool isAllocationCapped() override { return false; }
118 
119  void init();
120 
121  void getChunkMetadataVec(ChunkMetadataVector& chunkMetadataVec) override;
123  const ChunkKey& keyPrefix) override {
124  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
125  keyPrefix);
126  }
127 
132  void checkpoint() override;
133  void checkpoint(const int db_id, const int tb_id) override;
134 
139  inline size_t getNumReaderThreads() { return num_reader_threads_; }
140 
141  size_t getNumChunks() override;
142 
143  private:
144  AbstractBufferMgr* findFileMgr(const int db_id, const int tb_id);
145  void deleteFileMgr(const int db_id, const int tb_id);
146 
147  public:
148  AbstractBufferMgr* getFileMgr(const int db_id, const int tb_id);
149  AbstractBufferMgr* getFileMgr(const ChunkKey& key) {
150  return getFileMgr(key[0], key[1]);
151  }
152 
153  std::string getBasePath() const { return basePath_; }
154  size_t getDefaultPageSize() const { return defaultPageSize_; }
155 
156  void writeFileMgrData(FileMgr* fileMgr = 0);
157 
158  inline int getDBVersion() const { return mapd_db_version_; }
159  inline bool getDBConvert() const { return dbConvert_; }
160  inline void setDBConvert(bool val) { dbConvert_ = val; }
161 
162  void removeTableRelatedDS(const int db_id, const int tb_id) override;
163  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
164  size_t getTableEpoch(const int db_id, const int tb_id);
165 
166  private:
167  std::string basePath_;
169  int epoch_; /* the current epoch (time of last checkpoint) will be used for all
170  * tables except of the one for which the value of the epoch has been reset
171  * using --start-epoch option at start up to rollback this table's updates.
172  */
174  // bool isDirty_; /// true if metadata changed since last writeState()
175 
177  /* In future mapd_db_version_ may be added to AbstractBufferMgr class.
179  * This will allow support of different dbVersions for different tables, so
180  * original tables can be generated by different versions of mapd software.
181  */
182  bool dbConvert_;
183 
185  std::map<std::pair<int, int>, std::shared_ptr<AbstractBufferMgr>> ownedFileMgrs_;
186  std::map<std::pair<int, int>, AbstractBufferMgr*> allFileMgrs_;
187 
189 };
190 
191 } // namespace File_Namespace
192 
193 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
Definition: GlobalFileMgr.h:95
std::string getBasePath() const
bool isBufferOnDevice(const ChunkKey &key) override
Definition: GlobalFileMgr.h:63
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:57
int epoch_
number of threads used when loading data
#define LOG(tag)
Definition: Logger.h:188
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
AbstractBufferMgr * getFileMgr(const ChunkKey &key)
std::map< std::pair< int, int >, std::shared_ptr< AbstractBufferMgr > > ownedFileMgrs_
std::string getStringMgrType() override
size_t getAllocated() override
void init(LogOptions const &log_opts)
Definition: Logger.cpp:276
int mapd_db_version_
default page size, used to set FileMgr defaultPageSize_
std::map< std::pair< int, int >, AbstractBufferMgr * > allFileMgrs_
std::shared_timed_mutex mapd_shared_mutex
An AbstractBuffer is a unit of data management for a data manager.
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::string printSlabs() override
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:71
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:79
void free(AbstractBuffer *buffer) override
size_t num_reader_threads_
The OS file system path containing the files.
std::vector< int > ChunkKey
Definition: types.h:35
mapd_shared_mutex fileMgrs_mutex_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:83
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata > >> ChunkMetadataVector
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
size_t getInUseSize() override