OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
GlobalFileMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
27 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
28 
29 #include <iostream>
30 #include <map>
31 #include <mutex>
32 #include <set>
33 #include "../Shared/mapd_shared_mutex.h"
34 
35 #include "../AbstractBuffer.h"
36 #include "../AbstractBufferMgr.h"
37 #include "FileMgr.h"
38 
39 using namespace Data_Namespace;
40 
41 namespace File_Namespace {
42 
47 class GlobalFileMgr : public AbstractBufferMgr { // implements
48 
49  public:
51  GlobalFileMgr(const int deviceId,
52  std::string basePath = ".",
53  const size_t num_reader_threads = 0,
54  const size_t defaultPageSize = 2097152);
55 
58  size_t pageSize = 0,
59  const size_t numBytes = 0) override {
60  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
61  }
62 
63  bool isBufferOnDevice(const ChunkKey& key) override {
64  return getFileMgr(key)->isBufferOnDevice(key);
65  }
66 
68  // Purge == true means delete the data chunks -
69  // can't undelete and revert to previous
70  // state - reclaims disk space for chunk
71  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
72  return getFileMgr(key)->deleteBuffer(key, purge);
73  }
74 
75  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
76  const bool purge = true) override;
77 
79  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
80  return getFileMgr(key)->getBuffer(key, numBytes);
81  }
82 
83  void fetchBuffer(const ChunkKey& key,
84  AbstractBuffer* destBuffer,
85  const size_t numBytes) override {
86  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
87  }
88 
96  AbstractBuffer* d,
97  const size_t numBytes = 0) override {
98  return getFileMgr(key)->putBuffer(key, d, numBytes);
99  }
100 
101  // Buffer API
102  AbstractBuffer* alloc(const size_t numBytes) override {
103  LOG(FATAL) << "Operation not supported";
104  return nullptr; // satisfy return-type warning
105  }
106 
107  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }
108 
109  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
110  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
111  inline std::string printSlabs() override { return "Not Implemented"; }
112  inline void clearSlabs() override { /* noop */
113  }
114  inline size_t getMaxSize() override { return 0; }
115  inline size_t getInUseSize() override { return 0; }
116  inline size_t getAllocated() override { return 0; }
117  inline bool isAllocationCapped() override { return false; }
118 
119  void init();
120 
122  const ChunkKey& keyPrefix) override {
123  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
124  keyPrefix);
125  }
126 
131  void checkpoint() override;
132  void checkpoint(const int db_id, const int tb_id) override;
133 
138  inline size_t getNumReaderThreads() { return num_reader_threads_; }
139 
140  size_t getNumChunks() override;
141 
142  private:
143  AbstractBufferMgr* findFileMgr(const int db_id, const int tb_id);
144  void deleteFileMgr(const int db_id, const int tb_id);
145 
146  public:
147  AbstractBufferMgr* getFileMgr(const int db_id, const int tb_id);
148  AbstractBufferMgr* getFileMgr(const ChunkKey& key) {
149  return getFileMgr(key[0], key[1]);
150  }
151 
152  std::string getBasePath() const { return basePath_; }
153  size_t getDefaultPageSize() const { return defaultPageSize_; }
154 
155  void writeFileMgrData(FileMgr* fileMgr = 0);
156 
157  inline int getDBVersion() const { return mapd_db_version_; }
158  inline bool getDBConvert() const { return dbConvert_; }
159  inline void setDBConvert(bool val) { dbConvert_ = val; }
160 
161  void removeTableRelatedDS(const int db_id, const int tb_id) override;
162  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
163  size_t getTableEpoch(const int db_id, const int tb_id);
164 
165  // For testing purposes only
166  std::shared_ptr<FileMgr> getSharedFileMgr(const int db_id, const int table_id);
167 
168  // For testing purposes only
169  void setFileMgr(const int db_id, const int table_id, std::shared_ptr<FileMgr> file_mgr);
170 
171  private:
172  std::string basePath_;
174  int epoch_; /* the current epoch (time of last checkpoint) will be used for all
175  * tables except of the one for which the value of the epoch has been reset
176  * using --start-epoch option at start up to rollback this table's updates.
177  */
179  // bool isDirty_; /// true if metadata changed since last writeState()
180 
182  /* In future mapd_db_version_ may be added to AbstractBufferMgr class.
184  * This will allow support of different dbVersions for different tables, so
185  * original tables can be generated by different versions of mapd software.
186  */
187  bool dbConvert_;
188 
190  std::map<std::pair<int, int>, std::shared_ptr<FileMgr>> ownedFileMgrs_;
191  std::map<std::pair<int, int>, AbstractBufferMgr*> allFileMgrs_;
192 
194 };
195 
196 } // namespace File_Namespace
197 
198 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
Definition: GlobalFileMgr.h:95
std::vector< int > ChunkKey
Definition: types.h:37
std::string getBasePath() const
bool isBufferOnDevice(const ChunkKey &key) override
Definition: GlobalFileMgr.h:63
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:57
int epoch_
number of threads used when loading data
#define LOG(tag)
Definition: Logger.h:188
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
AbstractBufferMgr * getFileMgr(const ChunkKey &key)
std::string getStringMgrType() override
std::map< std::pair< int, int >, std::shared_ptr< FileMgr > > ownedFileMgrs_
size_t getAllocated() override
void init(LogOptions const &log_opts)
Definition: Logger.cpp:280
int mapd_db_version_
default page size, used to set FileMgr defaultPageSize_
std::map< std::pair< int, int >, AbstractBufferMgr * > allFileMgrs_
std::shared_timed_mutex mapd_shared_mutex
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::string printSlabs() override
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:71
size_t getDefaultPageSize() const
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:79
void free(AbstractBuffer *buffer) override
size_t num_reader_threads_
The OS file system path containing the files.
mapd_shared_mutex fileMgrs_mutex_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:83
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
size_t getInUseSize() override