OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
BufferMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #pragma once
26 
27 #define BOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED 1
28 
29 #include <iostream>
30 #include <list>
31 #include <map>
32 #include <mutex>
33 
34 #include "DataMgr/AbstractBuffer.h"
38 #include "Shared/types.h"
39 
40 class OutOfMemory : public std::runtime_error {
41  public:
42  OutOfMemory(size_t num_bytes)
43  : std::runtime_error(parse_error_str("OutOfMemory", num_bytes)) {
44  VLOG(1) << "Failed to allocate " << num_bytes << " bytes";
45  VLOG(1) << boost::stacktrace::stacktrace();
46  };
47 
48  OutOfMemory(const std::string& err) : std::runtime_error(parse_error_str(err, 0)) {
49  VLOG(1) << "Failed with OutOfMemory, condition " << err;
50  VLOG(1) << boost::stacktrace::stacktrace();
51  };
52 
53  OutOfMemory(const std::string& err, size_t num_bytes)
54  : std::runtime_error(parse_error_str(err, num_bytes)) {
55  VLOG(1) << "Failed to allocate " << num_bytes << " bytes with condition " << err;
56  VLOG(1) << boost::stacktrace::stacktrace();
57  };
58 
59  private:
60  std::string parse_error_str(const std::string& err, const size_t num_bytes = 0) {
61  if (num_bytes) {
62  return err + ": Failed to allocate " + std::to_string(num_bytes) + " bytes";
63  } else {
64  return "Failed to allocate memory with condition " + err;
65  }
66  }
67 };
68 
70  public:
71  FailedToCreateFirstSlab(size_t num_bytes)
72  : OutOfMemory("FailedToCreateFirstSlab", num_bytes) {}
73 };
74 
76  public:
77  FailedToCreateSlab(size_t num_bytes) : OutOfMemory("FailedToCreateSlab", num_bytes) {}
78 };
79 
80 class TooBigForSlab : public OutOfMemory {
81  public:
82  TooBigForSlab(size_t num_bytes) : OutOfMemory("TooBigForSlab", num_bytes) {}
83 };
84 
85 using namespace Data_Namespace;
86 
87 namespace Buffer_Namespace {
88 
96 class BufferMgr : public AbstractBufferMgr { // implements
97 
98  public:
100  BufferMgr(const int device_id,
101  const size_t max_buffer_size,
102  const size_t min_slab_size,
103  const size_t max_slab_size,
104  const size_t page_size,
105  AbstractBufferMgr* parent_mgr = 0);
106 
108  ~BufferMgr() override;
109  void reinit();
110 
111  std::string printSlab(size_t slab_num);
112  std::string printSlabs() override;
113 
114  void clearSlabs();
115  std::string printMap();
116  void printSegs();
117  std::string printSeg(BufferList::iterator& seg_it);
118  std::string keyToString(const ChunkKey& key);
119  size_t getInUseSize() override;
120  size_t getMaxSize() override;
121  size_t getAllocated() override;
122  size_t getMaxBufferSize();
123  size_t getMaxSlabSize();
124  size_t getPageSize();
125  bool isAllocationCapped() override;
126  const std::vector<BufferList>& getSlabSegments();
127 
129  AbstractBuffer* createBuffer(const ChunkKey& key,
130  const size_t page_size = 0,
131  const size_t initial_size = 0) override;
132 
134  void deleteBuffer(const ChunkKey& key, const bool purge = true) override;
135  void deleteBuffersWithPrefix(const ChunkKey& key_prefix,
136  const bool purge = true) override;
137 
139  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t num_bytes = 0) override;
140 
147  bool isBufferOnDevice(const ChunkKey& key) override;
148  void fetchBuffer(const ChunkKey& key,
149  AbstractBuffer* dest_buffer,
150  const size_t num_bytes = 0) override;
151  AbstractBuffer* putBuffer(const ChunkKey& key,
152  AbstractBuffer* d,
153  const size_t num_bytes = 0) override;
154  void checkpoint() override;
155  void checkpoint(const int db_id, const int tb_id) override;
156  void removeTableRelatedDS(const int db_id, const int table_id) override;
157 
158  // Buffer API
159  AbstractBuffer* alloc(const size_t num_bytes = 0) override;
160  void free(AbstractBuffer* buffer) override;
161 
163  size_t size();
164  size_t getNumChunks() override;
165 
166  BufferList::iterator reserveBuffer(BufferList::iterator& seg_it,
167  const size_t num_bytes);
168  void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector& chunk_metadata_vec,
169  const ChunkKey& key_prefix) override;
170 
171  protected:
172  const size_t
174  const size_t min_slab_size_;
175  const size_t max_slab_size_;
177  const size_t page_size_;
179  std::vector<int8_t*> slabs_;
180  std::vector<BufferList> slab_segments_;
182 
183  private:
184  BufferMgr(const BufferMgr&); // private copy constructor
185  BufferMgr& operator=(const BufferMgr&); // private assignment
186  void removeSegment(BufferList::iterator& seg_it);
187  BufferList::iterator findFreeBufferInSlab(const size_t slab_num,
188  const size_t num_pages_requested);
189  int getBufferId();
190  virtual void addSlab(const size_t slab_size) = 0;
191  virtual void freeAllMem() = 0;
192  virtual void allocateBuffer(BufferList::iterator seg_it,
193  const size_t page_size,
194  const size_t num_bytes) = 0;
195  void clear();
196 
197  std::mutex chunk_index_mutex_;
198  std::mutex sized_segs_mutex_;
200  std::mutex buffer_id_mutex_;
201  std::mutex global_mutex_;
202 
203  std::map<ChunkKey, BufferList::iterator> chunk_index_;
204  size_t max_buffer_pool_num_pages_; // max number of pages for buffer pool
210  AbstractBufferMgr* parent_mgr_;
212  unsigned int buffer_epoch_;
213 
215 
216  BufferList::iterator evict(BufferList::iterator& evict_start,
217  const size_t num_pages_requested,
218  const int slab_num);
232  BufferList::iterator findFreeBuffer(size_t num_bytes);
233 };
234 
235 } // namespace Buffer_Namespace
std::string parse_error_str(const std::string &err, const size_t num_bytes=0)
Definition: BufferMgr.h:60
AbstractBufferMgr * parent_mgr_
Definition: BufferMgr.h:210
std::vector< int > ChunkKey
Definition: types.h:37
const size_t max_buffer_pool_size_
Definition: BufferMgr.h:173
unsigned int buffer_epoch_
Definition: BufferMgr.h:212
OutOfMemory(size_t num_bytes)
Definition: BufferMgr.h:42
std::map< ChunkKey, BufferList::iterator > chunk_index_
Definition: BufferMgr.h:203
std::string to_string(char const *&&v)
const size_t min_slab_size_
max number of bytes allocated for the buffer pool
Definition: BufferMgr.h:174
Note(s): Forbid Copying Idiom 4.1.
Definition: BufferMgr.h:96
OutOfMemory(const std::string &err)
Definition: BufferMgr.h:48
TooBigForSlab(size_t num_bytes)
Definition: BufferMgr.h:82
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
OutOfMemory(const std::string &err, size_t num_bytes)
Definition: BufferMgr.h:53
std::mutex unsized_segs_mutex_
Definition: BufferMgr.h:199
FailedToCreateFirstSlab(size_t num_bytes)
Definition: BufferMgr.h:71
FailedToCreateSlab(size_t num_bytes)
Definition: BufferMgr.h:77
std::vector< int8_t * > slabs_
Definition: BufferMgr.h:179
std::list< BufferSeg > BufferList
Definition: BufferSeg.h:71
#define VLOG(n)
Definition: Logger.h:303