OmniSciDB  a987f07e93
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DataMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #ifndef DATAMGR_H
24 #define DATAMGR_H
25 
26 #include "../Shared/SystemParameters.h"
27 #include "../Shared/heavyai_shared_mutex.h"
28 #include "AbstractBuffer.h"
29 #include "AbstractBufferMgr.h"
30 #include "BufferMgr/Buffer.h"
31 #include "BufferMgr/BufferMgr.h"
32 #include "MemoryLevel.h"
33 #include "OSDependent/heavyai_fs.h"
35 
36 #include <fstream>
37 #include <iomanip>
38 #include <iostream>
39 #include <map>
40 #include <string>
41 #include <unordered_map>
42 #include <vector>
43 
44 namespace File_Namespace {
45 class FileBuffer;
46 class GlobalFileMgr;
47 } // namespace File_Namespace
48 
49 namespace CudaMgr_Namespace {
50 class CudaMgr;
51 }
52 
53 class DeviceAllocator;
54 
55 namespace Buffer_Namespace {
56 class CpuBufferMgr;
57 class GpuCudaBufferMgr;
58 } // namespace Buffer_Namespace
59 
60 namespace Data_Namespace {
61 
62 struct MemoryData {
63  size_t slabNum;
64  int32_t startPage;
65  size_t numPages;
66  uint32_t touch;
67  std::vector<int32_t> chunk_key;
69 };
70 
71 struct MemoryInfo {
72  size_t pageSize;
73  size_t maxNumPages;
76  std::vector<MemoryData> nodeMemoryData;
77 };
78 
81  std::unordered_map<std::string, size_t> items_;
82 
83  public:
85  std::ifstream f("/proc/meminfo");
86  std::stringstream ss;
87  ss << f.rdbuf();
88 
89  for (const std::string& line : split(ss.str(), "\n")) {
90  if (line.empty()) {
91  continue;
92  }
93  const auto nv = split(line, ":", 1);
94  CHECK(nv.size() == 2) << "unexpected line format in /proc/meminfo: " << line;
95  const auto name = strip(nv[0]), value = to_lower(strip(nv[1]));
96  auto v = split(value);
97  CHECK(v.size() == 1 || v.size() == 2)
98  << "unexpected line format in /proc/meminfo: " << line;
99  items_[name] = std::atoll(v[0].c_str());
100  if (v.size() == 2) {
101  CHECK(v[1] == "kb") << "unexpected unit suffix in /proc/meminfo: " << line;
102  items_[name] *= 1024;
103  }
104  }
105  }
106 
107  auto operator[](const std::string& name) { return items_[name]; }
108  auto begin() { return items_.begin(); }
109  auto end() { return items_.end(); }
110 };
111 
114  std::string inputText_;
115  std::vector<size_t> orders_;
117 
118  public:
119  ProcBuddyinfoParser(std::string text = {}) {
120  if (text.empty()) {
121  std::ifstream f("/proc/buddyinfo");
122  std::stringstream ss;
123  ss << f.rdbuf();
124  text = ss.str();
125  }
126  inputText_ = text;
127 
128  const size_t skipped_columns = 4;
129  // NOTE(sy): For now this calculation ignores the first four buddyinfo columns,
130  // but in the future we could break out subscores by node and/or by zone.
131  size_t number_of_columns = 0;
132  for (const std::string& line : split(text, "\n")) {
133  if (line.empty()) {
134  continue;
135  }
136  const auto columns = split(line);
137  CHECK_GT(columns.size(), skipped_columns) << "unexpected line format: " << line;
138  if (number_of_columns != 0) {
139  CHECK_EQ(columns.size(), number_of_columns)
140  << "expected line to have " << number_of_columns << " columns: " << line;
141  } else {
142  number_of_columns = columns.size();
143  orders_.resize(number_of_columns - skipped_columns, 0);
144  }
145  for (size_t i = skipped_columns; i < number_of_columns; ++i) {
146  orders_[i - skipped_columns] += strtoull(columns[i].c_str(), NULL, 10);
147  }
148  }
149 #ifdef __linux__
150  const long page_size =
151  sysconf(_SC_PAGE_SIZE); // in case x86-64 is configured to use 2MB pages
152 #else
153  const long page_size = heavyai::get_page_size();
154 #endif
155  size_t scaled = 0;
156  size_t total = 0;
157  for (size_t order = 0; order < orders_.size(); ++order) {
158  const size_t bytes = orders_[order] * (size_t(1) << order) * page_size;
159  scaled += (bytes * (orders_.size() - 1 - order)) / (orders_.size() - 1);
160  total += bytes;
161  }
162 
163  CHECK_GT(total, size_t(0)) << "failed to parse:\n" << text;
164  fragmentationPercent_ = (scaled * 100) / total;
165  }
166 
167  auto operator[](size_t order) { return orders_[order]; }
168  auto begin() { return orders_.begin(); }
169  auto end() { return orders_.end(); }
171  auto getInputText() { return inputText_; }
172 };
173 
174 class DataMgr {
175  friend class GlobalFileMgr;
176 
177  public:
178  explicit DataMgr(
179  const std::string& dataDir,
180  const SystemParameters& system_parameters,
181  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr,
182  const bool useGpus,
183  const size_t reservedGpuMem = (1 << 27),
184  const size_t numReaderThreads = 0, /* 0 means use default for # of reader threads */
185  const File_Namespace::DiskCacheConfig cacheConfig =
187  ~DataMgr();
189  const MemoryLevel memoryLevel,
190  const int deviceId = 0,
191  const size_t page_size = 0);
193  const MemoryLevel memoryLevel,
194  const int deviceId = 0,
195  const size_t numBytes = 0);
196  void deleteChunksWithPrefix(const ChunkKey& keyPrefix);
197  void deleteChunksWithPrefix(const ChunkKey& keyPrefix, const MemoryLevel memLevel);
198  AbstractBuffer* alloc(const MemoryLevel memoryLevel,
199  const int deviceId,
200  const size_t numBytes);
201  void free(AbstractBuffer* buffer);
202  // copies one buffer to another
203  void copy(AbstractBuffer* destBuffer, AbstractBuffer* srcBuffer);
204  bool isBufferOnDevice(const ChunkKey& key,
205  const MemoryLevel memLevel,
206  const int deviceId);
207  std::vector<MemoryInfo> getMemoryInfo(const MemoryLevel memLevel) const;
208  std::vector<MemoryInfo> getMemoryInfoUnlocked(const MemoryLevel memLevel) const;
209  std::string dumpLevel(const MemoryLevel memLevel);
210  void clearMemory(const MemoryLevel memLevel);
211 
212  const std::map<ChunkKey, File_Namespace::FileBuffer*>& getChunkMap();
213  void checkpoint(const int db_id,
214  const int tb_id); // checkpoint for individual table of DB
215  void checkpoint(const int db_id, const int table_id, const MemoryLevel memory_level);
217  const ChunkKey& keyPrefix);
218  inline bool gpusPresent() const { return hasGpus_; }
219  void removeTableRelatedDS(const int db_id, const int tb_id);
220  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
221  size_t getTableEpoch(const int db_id, const int tb_id);
222  void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id);
223 
224  CudaMgr_Namespace::CudaMgr* getCudaMgr() const { return cudaMgr_.get(); }
226  std::shared_ptr<ForeignStorageInterface> getForeignStorageInterface() const;
227 
228  // database_id, table_id, column_id, fragment_id
229  std::vector<int> levelSizes_;
230 
231  // std::unique_ptr<DeviceAllocator> createGpuAllocator(int device_id);
232  // NOTE(sy): Revisit how DataMgr should handle Cuda streams if Intel ever needs this.
233 
235  size_t free; // available CPU RAM memory in bytes
236  size_t total; // total CPU RAM memory in bytes
237  size_t resident; // resident process memory in bytes
238  size_t vtotal; // total process virtual memory in bytes
239  size_t regular; // process bytes non-shared
240  size_t shared; // process bytes shared (file maps + shmem)
241  size_t frag; // fragmentation percent
242  };
243 
245  static size_t getTotalSystemMemory();
246 
249  const size_t num_reader_threads,
250  const SystemParameters& sys_params);
251 
252  // Used for testing.
254 
255  // Used for testing.
256  Buffer_Namespace::GpuCudaBufferMgr* getGpuBufferMgr(int32_t device_id) const;
257 
258  private:
259  void populateMgrs(const SystemParameters& system_parameters,
260  const size_t userSpecifiedNumReaderThreads,
261  const File_Namespace::DiskCacheConfig& cache_config);
262  void convertDB(const std::string basePath);
263  void checkpoint(); // checkpoint for whole DB, called from convertDB proc only
264  void createTopLevelMetadata() const;
265  void allocateCpuBufferMgr(int32_t device_id,
266  size_t total_cpu_size,
267  size_t minCpuSlabSize,
268  size_t maxCpuSlabSize,
269  size_t page_size,
270  const std::vector<size_t>& cpu_tier_sizes);
271 
272  std::vector<std::vector<AbstractBufferMgr*>> bufferMgrs_;
273  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr_;
274  std::string dataDir_;
275  bool hasGpus_;
277  mutable std::mutex buffer_access_mutex_;
278 };
279 
280 std::ostream& operator<<(std::ostream& os, const DataMgr::SystemMemoryUsage&);
281 
282 } // namespace Data_Namespace
283 
284 #endif // DATAMGR_H
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:224
auto operator[](const std::string &name)
Definition: DataMgr.h:107
std::string to_lower(const std::string &str)
std::mutex buffer_access_mutex_
Definition: DataMgr.h:277
#define CHECK_EQ(x, y)
Definition: Logger.h:297
std::vector< int > ChunkKey
Definition: types.h:36
void resetPersistentStorage(const File_Namespace::DiskCacheConfig &cache_config, const size_t num_reader_threads, const SystemParameters &sys_params)
Definition: DataMgr.cpp:202
std::vector< MemoryData > nodeMemoryData
Definition: DataMgr.h:76
Buffer_Namespace::MemStatus memStatus
Definition: DataMgr.h:68
std::unordered_map< std::string, size_t > items_
Definition: DataMgr.h:81
ProcBuddyinfoParser(std::string text={})
Definition: DataMgr.h:119
std::vector< std::vector< AbstractBufferMgr * > > bufferMgrs_
Definition: DataMgr.h:272
std::vector< int > levelSizes_
Definition: DataMgr.h:229
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
std::ostream & operator<<(std::ostream &os, const DataMgr::SystemMemoryUsage &mem_info)
Definition: DataMgr.cpp:619
SystemMemoryUsage getSystemMemoryUsage() const
Definition: DataMgr.cpp:92
PersistentStorageMgr * getPersistentStorageMgr() const
Definition: DataMgr.cpp:634
void clearMemory(const MemoryLevel memLevel)
Definition: DataMgr.cpp:434
std::vector< MemoryInfo > getMemoryInfoUnlocked(const MemoryLevel memLevel) const
Definition: DataMgr.cpp:354
void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id)
Definition: DataMgr.cpp:599
std::string dumpLevel(const MemoryLevel memLevel)
Definition: DataMgr.cpp:418
void convertDB(const std::string basePath)
Definition: DataMgr.cpp:303
#define CHECK_GT(x, y)
Definition: Logger.h:301
constexpr double f
Definition: Utm.h:31
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
auto operator[](size_t order)
Definition: DataMgr.h:167
static size_t getTotalSystemMemory()
Definition: DataMgr.cpp:149
size_t getTableEpoch(const int db_id, const int tb_id)
Definition: DataMgr.cpp:592
Buffer_Namespace::GpuCudaBufferMgr * getGpuBufferMgr(int32_t device_id) const
Definition: DataMgr.cpp:643
std::shared_ptr< ForeignStorageInterface > getForeignStorageInterface() const
Definition: DataMgr.cpp:614
void createTopLevelMetadata() const
Definition: DataMgr.cpp:333
std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr_
Definition: DataMgr.h:273
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:466
void populateMgrs(const SystemParameters &system_parameters, const size_t userSpecifiedNumReaderThreads, const File_Namespace::DiskCacheConfig &cache_config)
Definition: DataMgr.cpp:216
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
const std::map< ChunkKey, File_Namespace::FileBuffer * > & getChunkMap()
An AbstractBuffer is a unit of data management for a data manager.
This file includes the class specification for the buffer manager (BufferMgr), and related data struc...
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:606
Parse /proc/meminfo into key/value pairs.
Definition: DataMgr.h:80
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:492
tuple line
Definition: parse_ast.py:10
bool isBufferOnDevice(const ChunkKey &key, const MemoryLevel memLevel, const int deviceId)
Definition: DataMgr.cpp:459
AbstractBuffer * getChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0)
Definition: DataMgr.cpp:481
std::vector< MemoryInfo > getMemoryInfo(const MemoryLevel memLevel) const
Definition: DataMgr.cpp:349
void removeTableRelatedDS(const int db_id, const int tb_id)
Definition: DataMgr.cpp:580
DataMgr(const std::string &dataDir, const SystemParameters &system_parameters, std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr, const bool useGpus, const size_t reservedGpuMem=(1<< 27), const size_t numReaderThreads=0, const File_Namespace::DiskCacheConfig cacheConfig=File_Namespace::DiskCacheConfig())
Definition: DataMgr.cpp:51
std::vector< size_t > orders_
Definition: DataMgr.h:115
Buffer_Namespace::CpuBufferMgr * getCpuBufferMgr() const
Definition: DataMgr.cpp:638
#define CHECK(condition)
Definition: Logger.h:289
void copy(AbstractBuffer *destBuffer, AbstractBuffer *srcBuffer)
Definition: DataMgr.cpp:531
bool gpusPresent() const
Definition: DataMgr.h:218
std::vector< int32_t > chunk_key
Definition: DataMgr.h:67
AbstractBuffer * createChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0)
Definition: DataMgr.cpp:472
void free(AbstractBuffer *buffer)
Definition: DataMgr.cpp:525
string name
Definition: setup.in.py:72
int get_page_size()
Definition: heavyai_fs.cpp:29
void allocateCpuBufferMgr(int32_t device_id, size_t total_cpu_size, size_t minCpuSlabSize, size_t maxCpuSlabSize, size_t page_size, const std::vector< size_t > &cpu_tier_sizes)
Definition: DataMgr.cpp:172
Parse /proc/buddyinfo into a Fragmentation health score.
Definition: DataMgr.h:113
void setTableEpoch(const int db_id, const int tb_id, const int start_epoch)
Definition: DataMgr.cpp:585
friend class GlobalFileMgr
Definition: DataMgr.h:175
AbstractBuffer * alloc(const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes)
Definition: DataMgr.cpp:516
std::string dataDir_
Definition: DataMgr.h:274