OmniSciDB  91042dcc5b
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DataMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
21 #ifndef DATAMGR_H
22 #define DATAMGR_H
23 
24 #include "../Shared/SystemParameters.h"
25 #include "../Shared/mapd_shared_mutex.h"
26 #include "AbstractBuffer.h"
27 #include "AbstractBufferMgr.h"
28 #include "BufferMgr/Buffer.h"
29 #include "BufferMgr/BufferMgr.h"
30 #include "MemoryLevel.h"
31 #include "OSDependent/omnisci_fs.h"
33 
34 #include <fstream>
35 #include <iomanip>
36 #include <iostream>
37 #include <map>
38 #include <string>
39 #include <unordered_map>
40 #include <vector>
41 
42 namespace File_Namespace {
43 class FileBuffer;
44 class GlobalFileMgr;
45 } // namespace File_Namespace
46 
47 namespace CudaMgr_Namespace {
48 class CudaMgr;
49 }
50 
51 class DeviceAllocator;
52 
53 namespace Buffer_Namespace {
54 class CpuBufferMgr;
55 class GpuCudaBufferMgr;
56 } // namespace Buffer_Namespace
57 
58 namespace Data_Namespace {
59 
60 struct MemoryData {
61  size_t slabNum;
62  int32_t startPage;
63  size_t numPages;
64  uint32_t touch;
65  std::vector<int32_t> chunk_key;
67 };
68 
69 struct MemoryInfo {
70  size_t pageSize;
71  size_t maxNumPages;
74  std::vector<MemoryData> nodeMemoryData;
75 };
76 
79  std::unordered_map<std::string, size_t> items_;
80 
81  public:
83  std::ifstream f("/proc/meminfo");
84  std::stringstream ss;
85  ss << f.rdbuf();
86 
87  for (const std::string& line : split(ss.str(), "\n")) {
88  if (line.empty()) {
89  continue;
90  }
91  const auto nv = split(line, ":", 1);
92  CHECK(nv.size() == 2) << "unexpected line format in /proc/meminfo: " << line;
93  const auto name = strip(nv[0]), value = to_lower(strip(nv[1]));
94  auto v = split(value);
95  CHECK(v.size() == 1 || v.size() == 2)
96  << "unexpected line format in /proc/meminfo: " << line;
97  items_[name] = std::atoll(v[0].c_str());
98  if (v.size() == 2) {
99  CHECK(v[1] == "kb") << "unexpected unit suffix in /proc/meminfo: " << line;
100  items_[name] *= 1024;
101  }
102  }
103  }
104 
105  auto operator[](const std::string& name) { return items_[name]; }
106  auto begin() { return items_.begin(); }
107  auto end() { return items_.end(); }
108 };
109 
112  std::string inputText_;
113  std::vector<size_t> orders_;
115 
116  public:
117  ProcBuddyinfoParser(std::string text = {}) {
118  if (text.empty()) {
119  std::ifstream f("/proc/buddyinfo");
120  std::stringstream ss;
121  ss << f.rdbuf();
122  text = ss.str();
123  }
124  inputText_ = text;
125 
126  const size_t skipped_columns = 4;
127  // NOTE(sy): For now this calculation ignores the first four buddyinfo columns,
128  // but in the future we could break out subscores by node and/or by zone.
129  size_t number_of_columns = 0;
130  for (const std::string& line : split(text, "\n")) {
131  if (line.empty()) {
132  continue;
133  }
134  const auto columns = split(line);
135  CHECK_GT(columns.size(), skipped_columns) << "unexpected line format: " << line;
136  if (number_of_columns != 0) {
137  CHECK_EQ(columns.size(), number_of_columns)
138  << "expected line to have " << number_of_columns << " columns: " << line;
139  } else {
140  number_of_columns = columns.size();
141  orders_.resize(number_of_columns - skipped_columns, 0);
142  }
143  for (size_t i = skipped_columns; i < number_of_columns; ++i) {
144  orders_[i - skipped_columns] += strtoull(columns[i].c_str(), NULL, 10);
145  }
146  }
147 #ifdef __linux__
148  const long page_size =
149  sysconf(_SC_PAGE_SIZE); // in case x86-64 is configured to use 2MB pages
150 #else
151  const long page_size = omnisci::get_page_size();
152 #endif
153  size_t scaled = 0;
154  size_t total = 0;
155  for (size_t order = 0; order < orders_.size(); ++order) {
156  const size_t bytes = orders_[order] * (size_t(1) << order) * page_size;
157  scaled += (bytes * (orders_.size() - 1 - order)) / (orders_.size() - 1);
158  total += bytes;
159  }
160 
161  CHECK_GT(total, size_t(0)) << "failed to parse:\n" << text;
162  fragmentationPercent_ = (scaled * 100) / total;
163  }
164 
165  auto operator[](size_t order) { return orders_[order]; }
166  auto begin() { return orders_.begin(); }
167  auto end() { return orders_.end(); }
169  auto getInputText() { return inputText_; }
170 };
171 
172 class DataMgr {
173  friend class GlobalFileMgr;
174 
175  public:
176  explicit DataMgr(
177  const std::string& dataDir,
178  const SystemParameters& system_parameters,
179  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr,
180  const bool useGpus,
181  const size_t reservedGpuMem = (1 << 27),
182  const size_t numReaderThreads = 0, /* 0 means use default for # of reader threads */
183  const File_Namespace::DiskCacheConfig cacheConfig =
185  ~DataMgr();
187  const MemoryLevel memoryLevel,
188  const int deviceId = 0,
189  const size_t page_size = 0);
191  const MemoryLevel memoryLevel,
192  const int deviceId = 0,
193  const size_t numBytes = 0);
194  void deleteChunksWithPrefix(const ChunkKey& keyPrefix);
195  void deleteChunksWithPrefix(const ChunkKey& keyPrefix, const MemoryLevel memLevel);
196  AbstractBuffer* alloc(const MemoryLevel memoryLevel,
197  const int deviceId,
198  const size_t numBytes);
199  void free(AbstractBuffer* buffer);
200  // copies one buffer to another
201  void copy(AbstractBuffer* destBuffer, AbstractBuffer* srcBuffer);
202  bool isBufferOnDevice(const ChunkKey& key,
203  const MemoryLevel memLevel,
204  const int deviceId);
205  std::vector<MemoryInfo> getMemoryInfo(const MemoryLevel memLevel) const;
206  std::vector<MemoryInfo> getMemoryInfoUnlocked(const MemoryLevel memLevel) const;
207  std::string dumpLevel(const MemoryLevel memLevel);
208  void clearMemory(const MemoryLevel memLevel);
209 
210  const std::map<ChunkKey, File_Namespace::FileBuffer*>& getChunkMap();
211  void checkpoint(const int db_id,
212  const int tb_id); // checkpoint for individual table of DB
213  void checkpoint(const int db_id, const int table_id, const MemoryLevel memory_level);
215  const ChunkKey& keyPrefix);
216  inline bool gpusPresent() const { return hasGpus_; }
217  void removeTableRelatedDS(const int db_id, const int tb_id);
218  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
219  size_t getTableEpoch(const int db_id, const int tb_id);
220  void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id);
221 
222  CudaMgr_Namespace::CudaMgr* getCudaMgr() const { return cudaMgr_.get(); }
224  std::shared_ptr<ForeignStorageInterface> getForeignStorageInterface() const;
225 
226  // database_id, table_id, column_id, fragment_id
227  std::vector<int> levelSizes_;
228 
229  std::unique_ptr<DeviceAllocator> createGpuAllocator(int device_id);
230 
232  size_t free; // available CPU RAM memory in bytes
233  size_t total; // total CPU RAM memory in bytes
234  size_t resident; // resident process memory in bytes
235  size_t vtotal; // total process virtual memory in bytes
236  size_t regular; // process bytes non-shared
237  size_t shared; // process bytes shared (file maps + shmem)
238  size_t frag; // fragmentation percent
239  };
240 
242  static size_t getTotalSystemMemory();
243 
246  const size_t num_reader_threads,
247  const SystemParameters& sys_params);
248 
249  // Used for testing.
251 
252  // Used for testing.
253  Buffer_Namespace::GpuCudaBufferMgr* getGpuBufferMgr(int32_t device_id) const;
254 
255  private:
256  void populateMgrs(const SystemParameters& system_parameters,
257  const size_t userSpecifiedNumReaderThreads,
258  const File_Namespace::DiskCacheConfig& cache_config);
259  void convertDB(const std::string basePath);
260  void checkpoint(); // checkpoint for whole DB, called from convertDB proc only
261  void createTopLevelMetadata() const;
262  void allocateCpuBufferMgr(int32_t device_id,
263  size_t total_cpu_size,
264  size_t minCpuSlabSize,
265  size_t maxCpuSlabSize,
266  size_t page_size,
267  const std::vector<size_t>& cpu_tier_sizes);
268 
269  std::vector<std::vector<AbstractBufferMgr*>> bufferMgrs_;
270  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr_;
271  std::string dataDir_;
272  bool hasGpus_;
274  mutable std::mutex buffer_access_mutex_;
275 };
276 
277 std::ostream& operator<<(std::ostream& os, const DataMgr::SystemMemoryUsage&);
278 
279 } // namespace Data_Namespace
280 
281 #endif // DATAMGR_H
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:222
auto operator[](const std::string &name)
Definition: DataMgr.h:105
std::string to_lower(const std::string &str)
std::mutex buffer_access_mutex_
Definition: DataMgr.h:274
#define CHECK_EQ(x, y)
Definition: Logger.h:219
std::vector< int > ChunkKey
Definition: types.h:37
void resetPersistentStorage(const File_Namespace::DiskCacheConfig &cache_config, const size_t num_reader_threads, const SystemParameters &sys_params)
Definition: DataMgr.cpp:202
std::vector< MemoryData > nodeMemoryData
Definition: DataMgr.h:74
Buffer_Namespace::MemStatus memStatus
Definition: DataMgr.h:66
std::unordered_map< std::string, size_t > items_
Definition: DataMgr.h:79
ProcBuddyinfoParser(std::string text={})
Definition: DataMgr.h:117
std::vector< std::vector< AbstractBufferMgr * > > bufferMgrs_
Definition: DataMgr.h:269
std::vector< int > levelSizes_
Definition: DataMgr.h:227
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
std::ostream & operator<<(std::ostream &os, const DataMgr::SystemMemoryUsage &mem_info)
Definition: DataMgr.cpp:629
SystemMemoryUsage getSystemMemoryUsage() const
Definition: DataMgr.cpp:92
PersistentStorageMgr * getPersistentStorageMgr() const
Definition: DataMgr.cpp:644
void clearMemory(const MemoryLevel memLevel)
Definition: DataMgr.cpp:435
string name
Definition: setup.in.py:72
std::vector< MemoryInfo > getMemoryInfoUnlocked(const MemoryLevel memLevel) const
Definition: DataMgr.cpp:355
void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id)
Definition: DataMgr.cpp:609
std::string dumpLevel(const MemoryLevel memLevel)
Definition: DataMgr.cpp:419
void convertDB(const std::string basePath)
Definition: DataMgr.cpp:303
#define CHECK_GT(x, y)
Definition: Logger.h:223
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
auto operator[](size_t order)
Definition: DataMgr.h:165
static size_t getTotalSystemMemory()
Definition: DataMgr.cpp:149
size_t getTableEpoch(const int db_id, const int tb_id)
Definition: DataMgr.cpp:602
Buffer_Namespace::GpuCudaBufferMgr * getGpuBufferMgr(int32_t device_id) const
Definition: DataMgr.cpp:653
std::shared_ptr< ForeignStorageInterface > getForeignStorageInterface() const
Definition: DataMgr.cpp:624
void createTopLevelMetadata() const
Definition: DataMgr.cpp:334
std::unique_ptr< DeviceAllocator > createGpuAllocator(int device_id)
Definition: DataMgr.cpp:526
std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr_
Definition: DataMgr.h:270
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:467
void populateMgrs(const SystemParameters &system_parameters, const size_t userSpecifiedNumReaderThreads, const File_Namespace::DiskCacheConfig &cache_config)
Definition: DataMgr.cpp:216
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
const std::map< ChunkKey, File_Namespace::FileBuffer * > & getChunkMap()
An AbstractBuffer is a unit of data management for a data manager.
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:616
Parse /proc/meminfo into key/value pairs.
Definition: DataMgr.h:78
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:493
tuple line
Definition: parse_ast.py:10
bool isBufferOnDevice(const ChunkKey &key, const MemoryLevel memLevel, const int deviceId)
Definition: DataMgr.cpp:460
AbstractBuffer * getChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0)
Definition: DataMgr.cpp:482
std::vector< MemoryInfo > getMemoryInfo(const MemoryLevel memLevel) const
Definition: DataMgr.cpp:350
void removeTableRelatedDS(const int db_id, const int tb_id)
Definition: DataMgr.cpp:590
DataMgr(const std::string &dataDir, const SystemParameters &system_parameters, std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr, const bool useGpus, const size_t reservedGpuMem=(1<< 27), const size_t numReaderThreads=0, const File_Namespace::DiskCacheConfig cacheConfig=File_Namespace::DiskCacheConfig())
Definition: DataMgr.cpp:51
std::vector< size_t > orders_
Definition: DataMgr.h:113
Buffer_Namespace::CpuBufferMgr * getCpuBufferMgr() const
Definition: DataMgr.cpp:648
#define CHECK(condition)
Definition: Logger.h:211
void copy(AbstractBuffer *destBuffer, AbstractBuffer *srcBuffer)
Definition: DataMgr.cpp:541
bool gpusPresent() const
Definition: DataMgr.h:216
char * f
std::vector< int32_t > chunk_key
Definition: DataMgr.h:65
AbstractBuffer * createChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0)
Definition: DataMgr.cpp:473
int get_page_size()
Definition: omnisci_fs.cpp:27
void free(AbstractBuffer *buffer)
Definition: DataMgr.cpp:535
void allocateCpuBufferMgr(int32_t device_id, size_t total_cpu_size, size_t minCpuSlabSize, size_t maxCpuSlabSize, size_t page_size, const std::vector< size_t > &cpu_tier_sizes)
Definition: DataMgr.cpp:172
Parse /proc/buddyinfo into a Fragmentation health score.
Definition: DataMgr.h:111
void setTableEpoch(const int db_id, const int tb_id, const int start_epoch)
Definition: DataMgr.cpp:595
friend class GlobalFileMgr
Definition: DataMgr.h:173
AbstractBuffer * alloc(const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes)
Definition: DataMgr.cpp:517
std::string dataDir_
Definition: DataMgr.h:271