OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
GlobalFileMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #ifndef DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
27 #define DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
28 
29 #include <iostream>
30 #include <map>
31 #include <mutex>
32 #include <set>
33 #include "../Shared/mapd_shared_mutex.h"
34 
35 #include "../AbstractBuffer.h"
36 #include "../AbstractBufferMgr.h"
37 #include "FileMgr.h"
38 
39 using namespace Data_Namespace;
40 
41 namespace File_Namespace {
42 
47 class GlobalFileMgr : public AbstractBufferMgr { // implements
48 
49  public:
51  GlobalFileMgr(const int deviceId,
52  std::string basePath = ".",
53  const size_t num_reader_threads = 0,
54  const size_t defaultPageSize = 2097152);
55 
57  ~GlobalFileMgr() override;
58 
61  size_t pageSize = 0,
62  const size_t numBytes = 0) override {
63  return getFileMgr(key)->createBuffer(key, pageSize, numBytes);
64  }
65 
66  bool isBufferOnDevice(const ChunkKey& key) override {
67  return getFileMgr(key)->isBufferOnDevice(key);
68  }
69 
71  // Purge == true means delete the data chunks -
72  // can't undelete and revert to previous
73  // state - reclaims disk space for chunk
74  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
75  return getFileMgr(key)->deleteBuffer(key, purge);
76  }
77 
78  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
79  const bool purge = true) override;
80 
82  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t numBytes = 0) override {
83  return getFileMgr(key)->getBuffer(key, numBytes);
84  }
85 
86  void fetchBuffer(const ChunkKey& key,
87  AbstractBuffer* destBuffer,
88  const size_t numBytes) override {
89  return getFileMgr(key)->fetchBuffer(key, destBuffer, numBytes);
90  }
91 
99  AbstractBuffer* d,
100  const size_t numBytes = 0) override {
101  return getFileMgr(key)->putBuffer(key, d, numBytes);
102  }
103 
104  // Buffer API
105  AbstractBuffer* alloc(const size_t numBytes) override {
106  LOG(FATAL) << "Operation not supported";
107  return nullptr; // satisfy return-type warning
108  }
109 
110  void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; }
111 
112  inline MgrType getMgrType() override { return GLOBAL_FILE_MGR; };
113  inline std::string getStringMgrType() override { return ToString(GLOBAL_FILE_MGR); }
114  inline std::string printSlabs() override { return "Not Implemented"; }
115  inline void clearSlabs() override { /* noop */
116  }
117  inline size_t getMaxSize() override { return 0; }
118  inline size_t getInUseSize() override { return 0; }
119  inline size_t getAllocated() override { return 0; }
120  inline bool isAllocationCapped() override { return false; }
121 
122  void init();
123 
124  void getChunkMetadataVec(
125  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec) override;
127  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec,
128  const ChunkKey& keyPrefix) override {
129  return getFileMgr(keyPrefix)->getChunkMetadataVecForKeyPrefix(chunkMetadataVec,
130  keyPrefix);
131  }
132 
137  void checkpoint() override;
138  void checkpoint(const int db_id, const int tb_id) override;
139 
144  inline size_t getNumReaderThreads() { return num_reader_threads_; }
145 
146  size_t getNumChunks() override;
147 
148  FileMgr* findFileMgr(const int db_id,
149  const int tb_id,
150  const bool removeFromMap = false);
151  FileMgr* getFileMgr(const int db_id, const int tb_id);
152  FileMgr* getFileMgr(const ChunkKey& key) { return getFileMgr(key[0], key[1]); }
153  std::string getBasePath() const { return basePath_; }
154  size_t getDefaultPageSize() const { return defaultPageSize_; }
155 
156  void writeFileMgrData(FileMgr* fileMgr = 0);
157 
158  inline int getDBVersion() const { return mapd_db_version_; }
159  inline bool getDBConvert() const { return dbConvert_; }
160  inline void setDBConvert(bool val) { dbConvert_ = val; }
161 
162  void removeTableRelatedDS(const int db_id, const int tb_id);
163  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
164  size_t getTableEpoch(const int db_id, const int tb_id);
165 
166  private:
167  std::string basePath_;
169  int epoch_; /* the current epoch (time of last checkpoint) will be used for all
170  * tables except of the one for which the value of the epoch has been reset
171  * using --start-epoch option at start up to rollback this table's updates.
172  */
174  // bool isDirty_; /// true if metadata changed since last writeState()
176  /* In future mapd_db_version_ may be added to AbstractBufferMgr class.
178  * This will allow support of different dbVersions for different tables, so
179  * original tables can be generated by different versions of mapd software.
180  */
181  bool dbConvert_;
182  std::map<std::pair<int, int>, FileMgr*> fileMgrs_;
185 };
186 
187 } // namespace File_Namespace
188 
189 #endif // DATAMGR_MEMORY_FILE_GLOBAL_FILEMGR_H
AbstractBuffer * putBuffer(const ChunkKey &key, AbstractBuffer *d, const size_t numBytes=0) override
Puts the contents of d into the Chunk with the given key.
Definition: GlobalFileMgr.h:98
std::vector< int > ChunkKey
Definition: types.h:35
std::string getBasePath() const
bool isBufferOnDevice(const ChunkKey &key) override
Definition: GlobalFileMgr.h:66
AbstractBuffer * createBuffer(const ChunkKey &key, size_t pageSize=0, const size_t numBytes=0) override
Creates a chunk with the specified key and page size.
Definition: GlobalFileMgr.h:60
int epoch_
number of threads used when loading data
#define LOG(tag)
Definition: Logger.h:185
bool isAllocationCapped() override
AbstractBuffer * alloc(const size_t numBytes) override
std::string getStringMgrType() override
size_t getAllocated() override
void init(LogOptions const &log_opts)
Definition: Logger.cpp:265
int mapd_db_version_
default page size, used to set FileMgr defaultPageSize_
std::shared_timed_mutex mapd_shared_mutex
void getChunkMetadataVecForKeyPrefix(std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec, const ChunkKey &keyPrefix) override
An AbstractBuffer is a unit of data management for a data manager.
FileMgr * getFileMgr(const ChunkKey &key)
std::string printSlabs() override
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
Deletes the chunk with the specified key.
Definition: GlobalFileMgr.h:74
size_t getDefaultPageSize() const
MgrType getMgrType() override
size_t getMaxSize() override
AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
Returns the a pointer to the chunk with the specified key.
Definition: GlobalFileMgr.h:82
void free(AbstractBuffer *buffer) override
size_t num_reader_threads_
The OS file system path containing the files.
mapd_shared_mutex fileMgrs_mutex_
void fetchBuffer(const ChunkKey &key, AbstractBuffer *destBuffer, const size_t numBytes) override
Definition: GlobalFileMgr.h:86
size_t getNumReaderThreads()
Returns number of threads defined by parameter num-reader-threads which should be used during initial...
size_t getInUseSize() override