26 #include "../Shared/SystemParameters.h"
27 #include "../Shared/heavyai_shared_mutex.h"
41 #include <unordered_map>
44 namespace File_Namespace {
49 namespace CudaMgr_Namespace {
55 namespace Buffer_Namespace {
57 class GpuCudaBufferMgr;
60 namespace Data_Namespace {
81 std::unordered_map<std::string, size_t>
items_;
85 std::ifstream
f(
"/proc/meminfo");
89 for (
const std::string&
line :
split(ss.str(),
"\n")) {
94 CHECK(nv.size() == 2) <<
"unexpected line format in /proc/meminfo: " <<
line;
96 auto v =
split(value);
97 CHECK(v.size() == 1 || v.size() == 2)
98 <<
"unexpected line format in /proc/meminfo: " <<
line;
101 CHECK(v[1] ==
"kb") <<
"unexpected unit suffix in /proc/meminfo: " <<
line;
102 items_[
name] *= 1024;
121 std::ifstream
f(
"/proc/buddyinfo");
122 std::stringstream ss;
128 const size_t skipped_columns = 4;
131 size_t number_of_columns = 0;
132 for (
const std::string&
line :
split(text,
"\n")) {
137 CHECK_GT(columns.size(), skipped_columns) <<
"unexpected line format: " <<
line;
138 if (number_of_columns != 0) {
139 CHECK_EQ(columns.size(), number_of_columns)
140 <<
"expected line to have " << number_of_columns <<
" columns: " <<
line;
142 number_of_columns = columns.size();
143 orders_.resize(number_of_columns - skipped_columns, 0);
145 for (
size_t i = skipped_columns; i < number_of_columns; ++i) {
146 orders_[i - skipped_columns] += strtoull(columns[i].c_str(), NULL, 10);
150 const long page_size =
151 sysconf(_SC_PAGE_SIZE);
157 for (
size_t order = 0; order <
orders_.size(); ++order) {
158 const size_t bytes =
orders_[order] * (size_t(1) << order) * page_size;
159 scaled += (bytes * (
orders_.size() - 1 - order)) / (
orders_.size() - 1);
163 CHECK_GT(total,
size_t(0)) <<
"failed to parse:\n" << text;
179 const std::string& dataDir,
181 std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr,
183 const size_t reservedGpuMem = (1 << 27),
184 const size_t numReaderThreads = 0,
190 const int deviceId = 0,
191 const size_t page_size = 0);
194 const int deviceId = 0,
195 const size_t numBytes = 0);
200 const size_t numBytes);
212 const std::map<ChunkKey, File_Namespace::FileBuffer*>&
getChunkMap();
220 void setTableEpoch(
const int db_id,
const int tb_id,
const int start_epoch);
249 const size_t num_reader_threads,
260 const size_t userSpecifiedNumReaderThreads,
262 void convertDB(
const std::string basePath);
266 size_t total_cpu_size,
267 size_t minCpuSlabSize,
268 size_t maxCpuSlabSize,
270 const std::vector<size_t>& cpu_tier_sizes);
273 std::unique_ptr<CudaMgr_Namespace::CudaMgr>
cudaMgr_;
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
auto operator[](const std::string &name)
std::mutex buffer_access_mutex_
std::vector< int > ChunkKey
void resetPersistentStorage(const File_Namespace::DiskCacheConfig &cache_config, const size_t num_reader_threads, const SystemParameters &sys_params)
std::vector< MemoryData > nodeMemoryData
Buffer_Namespace::MemStatus memStatus
std::unordered_map< std::string, size_t > items_
ProcBuddyinfoParser(std::string text={})
std::vector< std::vector< AbstractBufferMgr * > > bufferMgrs_
std::vector< int > levelSizes_
std::ostream & operator<<(std::ostream &os, const DataMgr::SystemMemoryUsage &mem_info)
SystemMemoryUsage getSystemMemoryUsage() const
PersistentStorageMgr * getPersistentStorageMgr() const
void clearMemory(const MemoryLevel memLevel)
std::vector< MemoryInfo > getMemoryInfoUnlocked(const MemoryLevel memLevel) const
void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id)
std::string dumpLevel(const MemoryLevel memLevel)
void convertDB(const std::string basePath)
auto operator[](size_t order)
static size_t getTotalSystemMemory()
size_t getTableEpoch(const int db_id, const int tb_id)
Buffer_Namespace::GpuCudaBufferMgr * getGpuBufferMgr(int32_t device_id) const
std::shared_ptr< ForeignStorageInterface > getForeignStorageInterface() const
void createTopLevelMetadata() const
size_t fragmentationPercent_
std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr_
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix)
void populateMgrs(const SystemParameters &system_parameters, const size_t userSpecifiedNumReaderThreads, const File_Namespace::DiskCacheConfig &cache_config)
const std::map< ChunkKey, File_Namespace::FileBuffer * > & getChunkMap()
An AbstractBuffer is a unit of data management for a data manager.
This file includes the class specification for the buffer manager (BufferMgr), and related data struc...
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Parse /proc/meminfo into key/value pairs.
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
bool isBufferOnDevice(const ChunkKey &key, const MemoryLevel memLevel, const int deviceId)
AbstractBuffer * getChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0)
std::vector< MemoryInfo > getMemoryInfo(const MemoryLevel memLevel) const
void removeTableRelatedDS(const int db_id, const int tb_id)
DataMgr(const std::string &dataDir, const SystemParameters &system_parameters, std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr, const bool useGpus, const size_t reservedGpuMem=(1<< 27), const size_t numReaderThreads=0, const File_Namespace::DiskCacheConfig cacheConfig=File_Namespace::DiskCacheConfig())
std::vector< size_t > orders_
Buffer_Namespace::CpuBufferMgr * getCpuBufferMgr() const
void copy(AbstractBuffer *destBuffer, AbstractBuffer *srcBuffer)
std::vector< int32_t > chunk_key
AbstractBuffer * createChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0)
void free(AbstractBuffer *buffer)
auto getFragmentationPercent()
void allocateCpuBufferMgr(int32_t device_id, size_t total_cpu_size, size_t minCpuSlabSize, size_t maxCpuSlabSize, size_t page_size, const std::vector< size_t > &cpu_tier_sizes)
Parse /proc/buddyinfo into a Fragmentation health score.
void setTableEpoch(const int db_id, const int tb_id, const int start_epoch)
friend class GlobalFileMgr
AbstractBuffer * alloc(const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes)