OmniSciDB  a47db9e897
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
DataMgr.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
21 #ifndef DATAMGR_H
22 #define DATAMGR_H
23 
24 #include "../Shared/MapDParameters.h"
25 #include "../Shared/mapd_shared_mutex.h"
26 #include "AbstractBuffer.h"
27 #include "AbstractBufferMgr.h"
28 #include "BufferMgr/Buffer.h"
29 #include "BufferMgr/BufferMgr.h"
30 #include "MemoryLevel.h"
31 
32 #include <iomanip>
33 #include <iostream>
34 #include <map>
35 #include <string>
36 #include <vector>
37 
38 namespace File_Namespace {
39 class FileBuffer;
40 class GlobalFileMgr;
41 } // namespace File_Namespace
42 
43 namespace CudaMgr_Namespace {
44 class CudaMgr;
45 }
46 
47 namespace Data_Namespace {
48 
49 struct MemoryData {
50  size_t slabNum;
51  int32_t startPage;
52  size_t numPages;
53  u_int32_t touch;
54  std::vector<int32_t> chunk_key;
56 };
57 
58 struct MemoryInfo {
59  size_t pageSize;
60  size_t maxNumPages;
63  std::vector<MemoryData> nodeMemoryData;
64 };
65 
66 class DataMgr {
67  friend class GlobalFileMgr;
68 
69  public:
70  DataMgr(const std::string& dataDir,
71  const MapDParameters& mapd_parameters,
72  const bool useGpus,
73  const int numGpus,
74  const int startGpu = 0,
75  const size_t reservedGpuMem = (1 << 27),
76  const size_t numReaderThreads =
77  0); /* 0 means use default for # of reader threads */
78  ~DataMgr();
80  const MemoryLevel memoryLevel,
81  const int deviceId = 0,
82  const size_t page_size = 0);
84  const MemoryLevel memoryLevel,
85  const int deviceId = 0,
86  const size_t numBytes = 0);
87  void deleteChunksWithPrefix(const ChunkKey& keyPrefix);
88  void deleteChunksWithPrefix(const ChunkKey& keyPrefix, const MemoryLevel memLevel);
89  AbstractBuffer* alloc(const MemoryLevel memoryLevel,
90  const int deviceId,
91  const size_t numBytes);
92  void free(AbstractBuffer* buffer);
93  void freeAllBuffers();
94  // copies one buffer to another
95  void copy(AbstractBuffer* destBuffer, AbstractBuffer* srcBuffer);
96  bool isBufferOnDevice(const ChunkKey& key,
97  const MemoryLevel memLevel,
98  const int deviceId);
99  std::vector<MemoryInfo> getMemoryInfo(const MemoryLevel memLevel);
100  std::string dumpLevel(const MemoryLevel memLevel);
101  void clearMemory(const MemoryLevel memLevel);
102 
103  // const std::map<ChunkKey, File_Namespace::FileBuffer *> & getChunkMap();
104  const std::map<ChunkKey, File_Namespace::FileBuffer*>& getChunkMap();
105  void checkpoint(const int db_id,
106  const int tb_id); // checkpoint for individual table of DB
107  void getChunkMetadataVec(
108  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec);
110  std::vector<std::pair<ChunkKey, ChunkMetadata>>& chunkMetadataVec,
111  const ChunkKey& keyPrefix);
112  inline bool gpusPresent() { return hasGpus_; }
113  void removeTableRelatedDS(const int db_id, const int tb_id);
114  void setTableEpoch(const int db_id, const int tb_id, const int start_epoch);
115  size_t getTableEpoch(const int db_id, const int tb_id);
116 
117  CudaMgr_Namespace::CudaMgr* getCudaMgr() const { return cudaMgr_.get(); }
119 
120  // database_id, table_id, column_id, fragment_id
121  std::vector<int> levelSizes_;
122 
123  private:
124  size_t getTotalSystemMemory();
125  void populateMgrs(const MapDParameters& mapd_parameters,
126  const size_t userSpecifiedNumReaderThreads);
127  void convertDB(const std::string basePath);
128  void checkpoint(); // checkpoint for whole DB, called from convertDB proc only
129  void createTopLevelMetadata() const;
130 
131  std::vector<std::vector<AbstractBufferMgr*>> bufferMgrs_;
132  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr_;
133  std::string dataDir_;
134  bool hasGpus_;
136  std::map<ChunkKey, std::shared_ptr<mapd_shared_mutex>> chunkMutexMap_;
138 };
139 } // namespace Data_Namespace
140 
141 #endif // DATAMGR_H
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:117
std::vector< int > ChunkKey
Definition: types.h:35
std::vector< MemoryData > nodeMemoryData
Definition: DataMgr.h:63
std::vector< std::vector< AbstractBufferMgr * > > bufferMgrs_
Definition: DataMgr.h:131
std::vector< int > levelSizes_
Definition: DataMgr.h:121
Buffer_Namespace::MemStatus isFree
Definition: DataMgr.h:55
void clearMemory(const MemoryLevel memLevel)
Definition: DataMgr.cpp:300
std::string dumpLevel(const MemoryLevel memLevel)
Definition: DataMgr.cpp:286
void convertDB(const std::string basePath)
Definition: DataMgr.cpp:139
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
size_t getTotalSystemMemory()
Definition: DataMgr.cpp:79
mapd_shared_mutex chunkMutexMapMutex_
Definition: DataMgr.h:137
size_t getTableEpoch(const int db_id, const int tb_id)
Definition: DataMgr.cpp:433
void createTopLevelMetadata() const
Definition: DataMgr.cpp:165
DataMgr(const std::string &dataDir, const MapDParameters &mapd_parameters, const bool useGpus, const int numGpus, const int startGpu=0, const size_t reservedGpuMem=(1<< 27), const size_t numReaderThreads=0)
Definition: DataMgr.cpp:46
std::shared_timed_mutex mapd_shared_mutex
std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr_
Definition: DataMgr.h:132
const std::map< ChunkKey, File_Namespace::FileBuffer * > & getChunkMap()
An AbstractBuffer is a unit of data management for a data manager.
std::vector< MemoryInfo > getMemoryInfo(const MemoryLevel memLevel)
Definition: DataMgr.cpp:176
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:354
bool isBufferOnDevice(const ChunkKey &key, const MemoryLevel memLevel, const int deviceId)
Definition: DataMgr.cpp:317
AbstractBuffer * getChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0)
Definition: DataMgr.cpp:344
void getChunkMetadataVecForKeyPrefix(std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec, const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:330
void removeTableRelatedDS(const int db_id, const int tb_id)
Definition: DataMgr.cpp:424
std::map< ChunkKey, std::shared_ptr< mapd_shared_mutex > > chunkMutexMap_
Definition: DataMgr.h:136
void copy(AbstractBuffer *destBuffer, AbstractBuffer *srcBuffer)
Definition: DataMgr.cpp:392
void populateMgrs(const MapDParameters &mapd_parameters, const size_t userSpecifiedNumReaderThreads)
Definition: DataMgr.cpp:98
std::vector< int32_t > chunk_key
Definition: DataMgr.h:54
AbstractBuffer * createChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0)
Definition: DataMgr.cpp:336
void free(AbstractBuffer *buffer)
Definition: DataMgr.cpp:382
void getChunkMetadataVec(std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec)
Definition: DataMgr.cpp:323
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:437
void setTableEpoch(const int db_id, const int tb_id, const int start_epoch)
Definition: DataMgr.cpp:428
friend class GlobalFileMgr
Definition: DataMgr.h:67
AbstractBuffer * alloc(const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes)
Definition: DataMgr.cpp:374
std::string dataDir_
Definition: DataMgr.h:133