OmniSciDB  21ac014ffc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
BufferMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #pragma once
26 
27 #define BOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED 1
28 
29 #include <iostream>
30 #include <list>
31 #include <map>
32 #include <mutex>
33 
34 #include "DataMgr/AbstractBuffer.h"
38 #include "Shared/types.h"
39 
40 class OutOfMemory : public std::runtime_error {
41  public:
42  OutOfMemory(size_t num_bytes)
43  : std::runtime_error(parse_error_str("OutOfMemory", num_bytes)) {
44  VLOG(1) << "Failed to allocate " << num_bytes << " bytes";
45  VLOG(1) << boost::stacktrace::stacktrace();
46  };
47 
48  OutOfMemory(const std::string& err) : std::runtime_error(parse_error_str(err, 0)) {
49  VLOG(1) << "Failed with OutOfMemory, condition " << err;
50  VLOG(1) << boost::stacktrace::stacktrace();
51  };
52 
53  OutOfMemory(const std::string& err, size_t num_bytes)
54  : std::runtime_error(parse_error_str(err, num_bytes)) {
55  VLOG(1) << "Failed to allocate " << num_bytes << " bytes with condition " << err;
56  VLOG(1) << boost::stacktrace::stacktrace();
57  };
58 
59  private:
60  std::string parse_error_str(const std::string& err, const size_t num_bytes = 0) {
61  if (num_bytes) {
62  return err + ": Failed to allocate " + std::to_string(num_bytes) + " bytes";
63  } else {
64  return "Failed to allocate memory with condition " + err;
65  }
66  }
67 };
68 
70  public:
71  FailedToCreateFirstSlab(size_t num_bytes)
72  : OutOfMemory("FailedToCreateFirstSlab", num_bytes) {}
73 };
74 
76  public:
77  FailedToCreateSlab(size_t num_bytes) : OutOfMemory("FailedToCreateSlab", num_bytes) {}
78 };
79 
80 class TooBigForSlab : public OutOfMemory {
81  public:
82  TooBigForSlab(size_t num_bytes) : OutOfMemory("TooBigForSlab", num_bytes) {}
83 };
84 
85 using namespace Data_Namespace;
86 
87 namespace Buffer_Namespace {
88 
96 class BufferMgr : public AbstractBufferMgr { // implements
97 
98  public:
100  BufferMgr(const int device_id,
101  const size_t max_buffer_size,
102  const size_t min_slab_size,
103  const size_t max_slab_size,
104  const size_t page_size,
105  AbstractBufferMgr* parent_mgr = 0);
106 
108  ~BufferMgr() override;
109  void reinit();
110 
111  void clear();
112 
113  std::string printSlab(size_t slab_num);
114  std::string printSlabs() override;
115  void clearSlabs() override;
116  std::string printMap();
117  void printSegs();
118  std::string printSeg(BufferList::iterator& seg_it);
119  std::string keyToString(const ChunkKey& key);
120  size_t getInUseSize() override;
121  size_t getMaxSize() override;
122  size_t getAllocated() override;
123  size_t getMaxBufferSize();
124  size_t getMaxSlabSize();
125  size_t getPageSize();
126  bool isAllocationCapped() override;
127  const std::vector<BufferList>& getSlabSegments();
128 
130  AbstractBuffer* createBuffer(const ChunkKey& key,
131  const size_t page_size = 0,
132  const size_t initial_size = 0) override;
133 
135  void deleteBuffer(const ChunkKey& key, const bool purge = true) override;
136  void deleteBuffersWithPrefix(const ChunkKey& key_prefix,
137  const bool purge = true) override;
138 
140  AbstractBuffer* getBuffer(const ChunkKey& key, const size_t num_bytes = 0) override;
141 
148  bool isBufferOnDevice(const ChunkKey& key) override;
149  void fetchBuffer(const ChunkKey& key,
150  AbstractBuffer* dest_buffer,
151  const size_t num_bytes = 0) override;
152  AbstractBuffer* putBuffer(const ChunkKey& key,
153  AbstractBuffer* d,
154  const size_t num_bytes = 0) override;
155  void checkpoint() override;
156  void checkpoint(const int db_id, const int tb_id) override;
157  void removeTableRelatedDS(const int db_id, const int table_id) override;
158 
159  // Buffer API
160  AbstractBuffer* alloc(const size_t num_bytes = 0) override;
161  void free(AbstractBuffer* buffer) override;
162 
164  size_t size();
165  size_t getNumChunks() override;
166 
167  BufferList::iterator reserveBuffer(BufferList::iterator& seg_it,
168  const size_t num_bytes);
169  void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector& chunk_metadata_vec,
170  const ChunkKey& key_prefix) override;
171 
172  protected:
173  const size_t
175  const size_t min_slab_size_;
176  const size_t max_slab_size_;
178  const size_t page_size_;
180  std::vector<int8_t*> slabs_;
181  std::vector<BufferList> slab_segments_;
183 
184  private:
185  BufferMgr(const BufferMgr&); // private copy constructor
186  BufferMgr& operator=(const BufferMgr&); // private assignment
187  void removeSegment(BufferList::iterator& seg_it);
188  BufferList::iterator findFreeBufferInSlab(const size_t slab_num,
189  const size_t num_pages_requested);
190  int getBufferId();
191  virtual void addSlab(const size_t slab_size) = 0;
192  virtual void freeAllMem() = 0;
193  virtual void allocateBuffer(BufferList::iterator seg_it,
194  const size_t page_size,
195  const size_t num_bytes) = 0;
196  std::mutex chunk_index_mutex_;
197  std::mutex sized_segs_mutex_;
199  std::mutex buffer_id_mutex_;
200  std::mutex global_mutex_;
201 
202  std::map<ChunkKey, BufferList::iterator> chunk_index_;
203  size_t max_buffer_pool_num_pages_; // max number of pages for buffer pool
209  AbstractBufferMgr* parent_mgr_;
211  unsigned int buffer_epoch_;
212 
214 
215  BufferList::iterator evict(BufferList::iterator& evict_start,
216  const size_t num_pages_requested,
217  const int slab_num);
231  BufferList::iterator findFreeBuffer(size_t num_bytes);
232 };
233 
234 } // namespace Buffer_Namespace
std::string parse_error_str(const std::string &err, const size_t num_bytes=0)
Definition: BufferMgr.h:60
AbstractBufferMgr * parent_mgr_
Definition: BufferMgr.h:209
std::vector< int > ChunkKey
Definition: types.h:37
const size_t max_buffer_pool_size_
Definition: BufferMgr.h:174
tuple d
Definition: test_fsi.py:9
unsigned int buffer_epoch_
Definition: BufferMgr.h:211
OutOfMemory(size_t num_bytes)
Definition: BufferMgr.h:42
std::map< ChunkKey, BufferList::iterator > chunk_index_
Definition: BufferMgr.h:202
std::string to_string(char const *&&v)
const size_t min_slab_size_
max number of bytes allocated for the buffer pool
Definition: BufferMgr.h:175
Note(s): Forbid Copying Idiom 4.1.
Definition: BufferMgr.h:96
OutOfMemory(const std::string &err)
Definition: BufferMgr.h:48
TooBigForSlab(size_t num_bytes)
Definition: BufferMgr.h:82
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
OutOfMemory(const std::string &err, size_t num_bytes)
Definition: BufferMgr.h:53
std::mutex unsized_segs_mutex_
Definition: BufferMgr.h:198
FailedToCreateFirstSlab(size_t num_bytes)
Definition: BufferMgr.h:71
FailedToCreateSlab(size_t num_bytes)
Definition: BufferMgr.h:77
std::vector< int8_t * > slabs_
Definition: BufferMgr.h:180
std::list< BufferSeg > BufferList
Definition: BufferSeg.h:71
#define VLOG(n)
Definition: Logger.h:300