24 #include "../Shared/SystemParameters.h"
25 #include "../Shared/mapd_shared_mutex.h"
37 #include <unordered_map>
40 namespace File_Namespace {
45 namespace CudaMgr_Namespace {
49 namespace Data_Namespace {
70 std::unordered_map<std::string, size_t>
items_;
74 std::ifstream
f(
"/proc/meminfo");
78 for (
const std::string&
line :
split(ss.str(),
"\n")) {
83 CHECK(nv.size() == 2) <<
"unexpected line format in /proc/meminfo: " <<
line;
85 auto v =
split(value);
86 CHECK(v.size() == 1 || v.size() == 2)
87 <<
"unexpected line format in /proc/meminfo: " <<
line;
90 CHECK(v[1] ==
"kb") <<
"unexpected unit suffix in /proc/meminfo: " <<
line;
110 std::ifstream
f(
"/proc/buddyinfo");
111 std::stringstream ss;
117 const size_t skipped_columns = 4;
120 size_t number_of_columns = 0;
121 for (
const std::string&
line :
split(text,
"\n")) {
126 CHECK_GT(columns.size(), skipped_columns) <<
"unexpected line format: " <<
line;
127 if (number_of_columns != 0) {
128 CHECK_EQ(columns.size(), number_of_columns)
129 <<
"expected line to have " << number_of_columns <<
" columns: " <<
line;
131 number_of_columns = columns.size();
132 orders_.resize(number_of_columns - skipped_columns, 0);
134 for (
size_t i = skipped_columns;
i < number_of_columns; ++
i) {
135 orders_[
i - skipped_columns] += strtoull(columns[
i].c_str(), NULL, 10);
139 const long page_size =
140 sysconf(_SC_PAGE_SIZE);
143 for (
size_t order = 0; order <
orders_.size(); ++order) {
144 const size_t bytes =
orders_[order] * (size_t(1) << order) * page_size;
145 scaled += (bytes * (
orders_.size() - 1 - order)) / (
orders_.size() - 1);
149 CHECK_GT(total,
size_t(0)) <<
"failed to parse:\n" << text;
165 const std::string& dataDir,
167 std::unique_ptr<CudaMgr_Namespace::CudaMgr> cudaMgr,
169 const size_t reservedGpuMem = (1 << 27),
170 const size_t numReaderThreads = 0,
175 const int deviceId = 0,
176 const size_t page_size = 0);
179 const int deviceId = 0,
180 const size_t numBytes = 0);
185 const size_t numBytes);
196 const std::map<ChunkKey, File_Namespace::FileBuffer*>&
getChunkMap();
203 void setTableEpoch(
const int db_id,
const int tb_id,
const int start_epoch);
227 const size_t num_reader_threads,
232 const size_t userSpecifiedNumReaderThreads,
234 void convertDB(
const std::string basePath);
239 std::unique_ptr<CudaMgr_Namespace::CudaMgr>
cudaMgr_;
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
auto operator[](const std::string &name)
std::mutex buffer_access_mutex_
std::vector< int > ChunkKey
std::vector< MemoryData > nodeMemoryData
Buffer_Namespace::MemStatus memStatus
std::unordered_map< std::string, size_t > items_
ProcBuddyinfoParser(std::string text={})
std::vector< std::vector< AbstractBufferMgr * > > bufferMgrs_
std::vector< int > levelSizes_
std::ostream & operator<<(std::ostream &os, const DataMgr::SystemMemoryUsage &mem_info)
SystemMemoryUsage getSystemMemoryUsage() const
void populateMgrs(const SystemParameters &system_parameters, const size_t userSpecifiedNumReaderThreads, const DiskCacheConfig &cache_config)
DataMgr(const std::string &dataDir, const SystemParameters &system_parameters, std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr, const bool useGpus, const size_t reservedGpuMem=(1<< 27), const size_t numReaderThreads=0, const DiskCacheConfig cacheConfig=DiskCacheConfig())
PersistentStorageMgr * getPersistentStorageMgr() const
void clearMemory(const MemoryLevel memLevel)
std::string dumpLevel(const MemoryLevel memLevel)
void convertDB(const std::string basePath)
auto operator[](size_t order)
static size_t getTotalSystemMemory()
size_t getTableEpoch(const int db_id, const int tb_id)
void createTopLevelMetadata() const
size_t fragmentationPercent_
std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr_
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix)
const std::map< ChunkKey, File_Namespace::FileBuffer * > & getChunkMap()
An AbstractBuffer is a unit of data management for a data manager.
std::vector< MemoryInfo > getMemoryInfo(const MemoryLevel memLevel)
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Parse /proc/meminfo into key/value pairs.
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
bool isBufferOnDevice(const ChunkKey &key, const MemoryLevel memLevel, const int deviceId)
AbstractBuffer * getChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0)
void removeTableRelatedDS(const int db_id, const int tb_id)
std::vector< size_t > orders_
void resetPersistentStorage(const DiskCacheConfig &cache_config, const size_t num_reader_threads, const SystemParameters &sys_params)
void copy(AbstractBuffer *destBuffer, AbstractBuffer *srcBuffer)
std::vector< int32_t > chunk_key
AbstractBuffer * createChunkBuffer(const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0)
void free(AbstractBuffer *buffer)
auto getFragmentationPercent()
Parse /proc/buddyinfo into a Fragmentation health score.
void setTableEpoch(const int db_id, const int tb_id, const int start_epoch)
friend class GlobalFileMgr
AbstractBuffer * alloc(const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes)