OmniSciDB  c07336695a
DataMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
21 #ifndef DATAMGR_H
22 #define DATAMGR_H
23 
24 #include "../Shared/MapDParameters.h"
25 #include "../Shared/mapd_shared_mutex.h"
26 #include "AbstractBuffer.h"
27 #include "AbstractBufferMgr.h"
28 #include "BufferMgr/Buffer.h"
29 #include "BufferMgr/BufferMgr.h"
30 #include "MemoryLevel.h"
31 
32 #include <iomanip>
33 #include <iostream>
34 #include <map>
35 #include <string>
36 #include <vector>
37 
38 namespace File_Namespace {
39 class FileBuffer;
40 }
41 
42 namespace CudaMgr_Namespace {
43 class CudaMgr;
44 }
45 
46 namespace Data_Namespace {
47 
48 struct MemoryData {
49  size_t slabNum;
50  int32_t startPage;
51  size_t numPages;
52  u_int32_t touch;
53  std::vector<int32_t> chunk_key;
55 };
56 
57 struct MemoryInfo {
58  size_t pageSize;
59  size_t maxNumPages;
62  std::vector<MemoryData> nodeMemoryData;
63 };
64 
65 class DataMgr {
66  friend class GlobalFileMgr;
67 
68  public:
69  DataMgr(const std::string& dataDir,
70  const MapDParameters& mapd_parameters,
71  const bool useGpus,
72  const int numGpus,
73  const int startGpu = 0,
74  const size_t reservedGpuMem = (1 << 27),
75  const size_t numReaderThreads =
76  0); /* 0 means use default for # of reader threads */
77  ~DataMgr();
78  AbstractBuffer* createChunkBuffer(const ChunkKey& key,
79  const MemoryLevel memoryLevel,
80  const int deviceId = 0,
81  const size_t page_size = 0);
82  AbstractBuffer* getChunkBuffer(const ChunkKey& key,
83  const MemoryLevel memoryLevel,
84  const int deviceId = 0,
85  const size_t numBytes = 0);
86  void deleteChunksWithPrefix(const ChunkKey& keyPrefix);
87  void deleteChunksWithPrefix(const ChunkKey& keyPrefix, const MemoryLevel memLevel);
88  AbstractBuffer* alloc(const MemoryLevel memoryLevel,
89  const int deviceId,
90  const size_t numBytes);
91  void free(AbstractBuffer* buffer);
92  void freeAllBuffers();
93  // copies one buffer to another
94  void copy(AbstractBuffer* destBuffer, AbstractBuffer* srcBuffer);
95  bool isBufferOnDevice(const ChunkKey& key,
96  const MemoryLevel memLevel,
97  const int deviceId);
98  std::vector<MemoryInfo> getMemoryInfo(const MemoryLevel memLevel);
99  std::string dumpLevel(const MemoryLevel memLevel);
100  void clearMemory(const MemoryLevel memLevel);
101 
102  // const std::map<ChunkKey, File_Namespace::FileBuffer *> & getChunkMap();
103  const std::map<ChunkKey, File_Namespace::FileBuffer*>& getChunkMap();
104  void checkpoint(const int db_id,
105  const int tb_id); // checkpoint for individual table of DB
106  void getChunkMetadataVec(
107  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec);
108  void getChunkMetadataVecForKeyPrefix(
109  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec,
110  const ChunkKey& keyPrefix);
111  inline bool gpusPresent() { return hasGpus_; }
112  void removeTableRelatedDS(const int db_id, const int tb_id);
113  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
114  size_t getTableEpoch(const int db_id, const int tb_id);
115 
116  CudaMgr_Namespace::CudaMgr* getCudaMgr() const { return cudaMgr_.get(); }
117 
118  // database_id, table_id, column_id, fragment_id
119  std::vector<int> levelSizes_;
120 
121  private:
122  size_t getTotalSystemMemory();
123  void populateMgrs(const MapDParameters& mapd_parameters,
124  const size_t userSpecifiedNumReaderThreads);
125  void convertDB(const std::string basePath);
126  void checkpoint(); // checkpoint for whole DB, called from convertDB proc only
127  void createTopLevelMetadata() const;
128 
129  std::vector<std::vector<AbstractBufferMgr*>> bufferMgrs_;
130  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr_;
131  std::string dataDir_;
132  bool hasGpus_;
134  std::map<ChunkKey, std::shared_ptr<mapd_shared_mutex>> chunkMutexMap_;
136 };
137 } // namespace Data_Namespace
138 
139 #endif // DATAMGR_H
std::vector< MemoryData > nodeMemoryData
Definition: DataMgr.h:62
std::vector< std::vector< AbstractBufferMgr * > > bufferMgrs_
Definition: DataMgr.h:129
std::vector< int > levelSizes_
Definition: DataMgr.h:119
Buffer_Namespace::MemStatus isFree
Definition: DataMgr.h:54
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
mapd_shared_mutex chunkMutexMapMutex_
Definition: DataMgr.h:135
std::shared_timed_mutex mapd_shared_mutex
std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr_
Definition: DataMgr.h:130
An AbstractBuffer is a unit of data management for a data manager.
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:116
std::map< ChunkKey, std::shared_ptr< mapd_shared_mutex > > chunkMutexMap_
Definition: DataMgr.h:134
std::vector< int > ChunkKey
Definition: types.h:35
std::vector< int32_t > chunk_key
Definition: DataMgr.h:53
std::string dataDir_
Definition: DataMgr.h:131