OmniSciDB  0b528656ed
DataMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
21 #ifndef DATAMGR_H
22 #define DATAMGR_H
23 
24 #include "../Shared/SystemParameters.h"
25 #include "../Shared/mapd_shared_mutex.h"
26 #include "AbstractBuffer.h"
27 #include "AbstractBufferMgr.h"
28 #include "BufferMgr/Buffer.h"
29 #include "BufferMgr/BufferMgr.h"
31 #include "MemoryLevel.h"
32 
33 #include <iomanip>
34 #include <iostream>
35 #include <map>
36 #include <string>
37 #include <unordered_map>
38 #include <vector>
39 
40 namespace File_Namespace {
41 class FileBuffer;
42 class GlobalFileMgr;
43 } // namespace File_Namespace
44 
45 namespace CudaMgr_Namespace {
46 class CudaMgr;
47 }
48 
49 namespace Data_Namespace {
50 
51 struct MemoryData {
52  size_t slabNum;
53  int32_t startPage;
54  size_t numPages;
55  u_int32_t touch;
56  std::vector<int32_t> chunk_key;
58 };
59 
60 struct MemoryInfo {
61  size_t pageSize;
62  size_t maxNumPages;
65  std::vector<MemoryData> nodeMemoryData;
66 };
67 
70  std::unordered_map<std::string, size_t> items_;
71 
72  public:
74  std::ifstream f("/proc/meminfo");
75  std::stringstream ss;
76  ss << f.rdbuf();
77  for (const std::string& line : split(ss.str(), "\n")) {
78  if (line.empty()) {
79  continue;
80  }
81  const auto nv = split(line, ":", 1);
82  CHECK(nv.size() == 2) << "unexpected line format in /proc/meminfo: " << line;
83  const auto name = strip(nv[0]), value = to_lower(strip(nv[1]));
84  auto v = split(value);
85  CHECK(v.size() == 1 || v.size() == 2)
86  << "unexpected line format in /proc/meminfo: " << line;
87  items_[name] = std::atoll(v[0].c_str());
88  if (v.size() == 2) {
89  CHECK(v[1] == "kb") << "unexpected unit suffix in /proc/meminfo: " << line;
90  items_[name] *= 1024;
91  }
92  }
93  }
94  auto operator[](const std::string& name) { return items_[name]; }
95  auto begin() { return items_.begin(); }
96  auto end() { return items_.end(); }
97 };
98 
99 class DataMgr {
100  friend class GlobalFileMgr;
101 
102  public:
103  DataMgr(const std::string& dataDir,
104  const SystemParameters& system_parameters,
105  const bool useGpus,
106  const int numGpus,
107  const int startGpu = 0,
108  const size_t reservedGpuMem = (1 << 27),
109  const size_t numReaderThreads =
110  0); /* 0 means use default for # of reader threads */
111  ~DataMgr();
112  AbstractBuffer* createChunkBuffer(const ChunkKey& key,
113  const MemoryLevel memoryLevel,
114  const int deviceId = 0,
115  const size_t page_size = 0);
116  AbstractBuffer* getChunkBuffer(const ChunkKey& key,
117  const MemoryLevel memoryLevel,
118  const int deviceId = 0,
119  const size_t numBytes = 0);
120  void deleteChunksWithPrefix(const ChunkKey& keyPrefix);
121  void deleteChunksWithPrefix(const ChunkKey& keyPrefix, const MemoryLevel memLevel);
122  AbstractBuffer* alloc(const MemoryLevel memoryLevel,
123  const int deviceId,
124  const size_t numBytes);
125  void free(AbstractBuffer* buffer);
126  // copies one buffer to another
127  void copy(AbstractBuffer* destBuffer, AbstractBuffer* srcBuffer);
128  bool isBufferOnDevice(const ChunkKey& key,
129  const MemoryLevel memLevel,
130  const int deviceId);
131  std::vector<MemoryInfo> getMemoryInfo(const MemoryLevel memLevel);
132  std::string dumpLevel(const MemoryLevel memLevel);
133  void clearMemory(const MemoryLevel memLevel);
134 
135  const std::map<ChunkKey, File_Namespace::FileBuffer*>& getChunkMap();
136  void checkpoint(const int db_id,
137  const int tb_id); // checkpoint for individual table of DB
138  void getChunkMetadataVec(ChunkMetadataVector& chunkMetadataVec);
139  void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector& chunkMetadataVec,
140  const ChunkKey& keyPrefix);
141  inline bool gpusPresent() { return hasGpus_; }
142  void removeTableRelatedDS(const int db_id, const int tb_id);
143  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
144  size_t getTableEpoch(const int db_id, const int tb_id);
145 
146  CudaMgr_Namespace::CudaMgr* getCudaMgr() const { return cudaMgr_.get(); }
147  File_Namespace::GlobalFileMgr* getGlobalFileMgr() const;
148 
149  // database_id, table_id, column_id, fragment_id
150  std::vector<int> levelSizes_;
151 
153  int64_t free; // available CPU RAM memory in bytes
154  int64_t total; // total CPU RAM memory in bytes
155  int64_t resident; // resident process memory in bytes
156  int64_t vtotal; // total process virtual memory in bytes
157  int64_t regular; // process bytes non-shared
158  int64_t shared; // process bytes shared (file maps + shmem)
159  };
160 
161  SystemMemoryUsage getSystemMemoryUsage() const;
162  static size_t getTotalSystemMemory();
163 
164  foreign_storage::ForeignStorageMgr* getForeignStorageMgr() const;
165 
166  private:
167  void populateMgrs(const SystemParameters& system_parameters,
168  const size_t userSpecifiedNumReaderThreads);
169  void convertDB(const std::string basePath);
170  void checkpoint(); // checkpoint for whole DB, called from convertDB proc only
171  void createTopLevelMetadata() const;
172 
173  std::vector<std::vector<AbstractBufferMgr*>> bufferMgrs_;
174  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr_;
175  std::string dataDir_;
176  bool hasGpus_;
179 };
180 
181 std::ostream& operator<<(std::ostream& os, const DataMgr::SystemMemoryUsage&);
182 
183 } // namespace Data_Namespace
184 
185 #endif // DATAMGR_H
auto operator[](const std::string &name)
Definition: DataMgr.h:94
std::string to_lower(const std::string &str)
std::mutex buffer_access_mutex_
Definition: DataMgr.h:178
std::vector< MemoryData > nodeMemoryData
Definition: DataMgr.h:65
Buffer_Namespace::MemStatus memStatus
Definition: DataMgr.h:57
std::unordered_map< std::string, size_t > items_
Definition: DataMgr.h:70
std::vector< std::vector< AbstractBufferMgr * > > bufferMgrs_
Definition: DataMgr.h:173
std::vector< int > levelSizes_
Definition: DataMgr.h:150
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
std::ostream & operator<<(std::ostream &os, const SessionInfo &session_info)
Definition: SessionInfo.cpp:53
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr_
Definition: DataMgr.h:174
An AbstractBuffer is a unit of data management for a data manager.
Parse /proc/meminfo into key/value pairs.
Definition: DataMgr.h:69
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:146
#define CHECK(condition)
Definition: Logger.h:197
std::vector< int > ChunkKey
Definition: types.h:35
std::vector< int32_t > chunk_key
Definition: DataMgr.h:56
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata > >> ChunkMetadataVector
std::string dataDir_
Definition: DataMgr.h:175