OmniSciDB
c1a53651b2
|
#include <DataMgr.h>
Classes | |
struct | SystemMemoryUsage |
Public Member Functions | |
DataMgr (const std::string &dataDir, const SystemParameters &system_parameters, std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr, const bool useGpus, const size_t reservedGpuMem=(1<< 27), const size_t numReaderThreads=0, const File_Namespace::DiskCacheConfig cacheConfig=File_Namespace::DiskCacheConfig()) | |
~DataMgr () | |
AbstractBuffer * | createChunkBuffer (const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0) |
AbstractBuffer * | getChunkBuffer (const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0) |
void | deleteChunksWithPrefix (const ChunkKey &keyPrefix) |
void | deleteChunksWithPrefix (const ChunkKey &keyPrefix, const MemoryLevel memLevel) |
AbstractBuffer * | alloc (const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes) |
void | free (AbstractBuffer *buffer) |
void | copy (AbstractBuffer *destBuffer, AbstractBuffer *srcBuffer) |
bool | isBufferOnDevice (const ChunkKey &key, const MemoryLevel memLevel, const int deviceId) |
std::vector< MemoryInfo > | getMemoryInfo (const MemoryLevel memLevel) const |
std::vector< MemoryInfo > | getMemoryInfoUnlocked (const MemoryLevel memLevel) const |
std::string | dumpLevel (const MemoryLevel memLevel) |
void | clearMemory (const MemoryLevel memLevel) |
const std::map< ChunkKey, File_Namespace::FileBuffer * > & | getChunkMap () |
void | checkpoint (const int db_id, const int tb_id) |
void | checkpoint (const int db_id, const int table_id, const MemoryLevel memory_level) |
void | getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) |
bool | gpusPresent () const |
void | removeTableRelatedDS (const int db_id, const int tb_id) |
void | setTableEpoch (const int db_id, const int tb_id, const int start_epoch) |
size_t | getTableEpoch (const int db_id, const int tb_id) |
void | resetTableEpochFloor (const int32_t db_id, const int32_t tb_id) |
CudaMgr_Namespace::CudaMgr * | getCudaMgr () const |
File_Namespace::GlobalFileMgr * | getGlobalFileMgr () const |
std::shared_ptr < ForeignStorageInterface > | getForeignStorageInterface () const |
SystemMemoryUsage | getSystemMemoryUsage () const |
PersistentStorageMgr * | getPersistentStorageMgr () const |
void | resetPersistentStorage (const File_Namespace::DiskCacheConfig &cache_config, const size_t num_reader_threads, const SystemParameters &sys_params) |
Buffer_Namespace::CpuBufferMgr * | getCpuBufferMgr () const |
Buffer_Namespace::GpuCudaBufferMgr * | getGpuBufferMgr (int32_t device_id) const |
Static Public Member Functions | |
static size_t | getTotalSystemMemory () |
Public Attributes | |
std::vector< int > | levelSizes_ |
Private Member Functions | |
void | populateMgrs (const SystemParameters &system_parameters, const size_t userSpecifiedNumReaderThreads, const File_Namespace::DiskCacheConfig &cache_config) |
void | convertDB (const std::string basePath) |
void | checkpoint () |
void | createTopLevelMetadata () const |
void | allocateCpuBufferMgr (int32_t device_id, size_t total_cpu_size, size_t minCpuSlabSize, size_t maxCpuSlabSize, size_t page_size, const std::vector< size_t > &cpu_tier_sizes) |
Private Attributes | |
std::vector< std::vector < AbstractBufferMgr * > > | bufferMgrs_ |
std::unique_ptr < CudaMgr_Namespace::CudaMgr > | cudaMgr_ |
std::string | dataDir_ |
bool | hasGpus_ |
size_t | reservedGpuMem_ |
std::mutex | buffer_access_mutex_ |
Friends | |
class | GlobalFileMgr |
|
explicit |
Definition at line 51 of file DataMgr.cpp.
Data_Namespace::DataMgr::~DataMgr | ( | ) |
Definition at line 83 of file DataMgr.cpp.
References bufferMgrs_.
AbstractBuffer * Data_Namespace::DataMgr::alloc | ( | const MemoryLevel | memoryLevel, |
const int | deviceId, | ||
const size_t | numBytes | ||
) |
Definition at line 516 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK_LT, and levelSizes_.
Referenced by ThrustAllocator::allocate(), ThrustAllocator::allocateScopedBuffer(), CudaAllocator::allocGpuAbstractBuffer(), and InValuesBitmap::InValuesBitmap().
|
private |
Definition at line 172 of file DataMgr.cpp.
References bufferMgrs_, and cudaMgr_.
Referenced by populateMgrs().
void Data_Namespace::DataMgr::checkpoint | ( | const int | db_id, |
const int | tb_id | ||
) |
Definition at line 545 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by UpdelRoll::stageUpdate().
void Data_Namespace::DataMgr::checkpoint | ( | const int | db_id, |
const int | table_id, | ||
const MemoryLevel | memory_level | ||
) |
Definition at line 557 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK_LT, and levelSizes_.
|
private |
Definition at line 568 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by convertDB().
void Data_Namespace::DataMgr::clearMemory | ( | const MemoryLevel | memLevel | ) |
Definition at line 434 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK, cudaMgr_, Data_Namespace::GPU_LEVEL, logger::INFO, LOG, and logger::WARNING.
Referenced by Executor::clearMemory().
|
private |
Definition at line 303 of file DataMgr.cpp.
References bufferMgrs_, CHECK, checkpoint(), logger::FATAL, getGlobalFileMgr(), logger::INFO, shared::kDataDirectoryName, and LOG.
void Data_Namespace::DataMgr::copy | ( | AbstractBuffer * | destBuffer, |
AbstractBuffer * | srcBuffer | ||
) |
Definition at line 531 of file DataMgr.cpp.
References Data_Namespace::AbstractBuffer::getDeviceId(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::getType(), Data_Namespace::AbstractBuffer::size(), and Data_Namespace::AbstractBuffer::write().
AbstractBuffer * Data_Namespace::DataMgr::createChunkBuffer | ( | const ChunkKey & | key, |
const MemoryLevel | memoryLevel, | ||
const int | deviceId = 0 , |
||
const size_t | page_size = 0 |
||
) |
Definition at line 472 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by Chunk_NS::Chunk::createChunkBuffer().
|
private |
Definition at line 333 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
Referenced by resetPersistentStorage().
void Data_Namespace::DataMgr::deleteChunksWithPrefix | ( | const ChunkKey & | keyPrefix | ) |
Definition at line 492 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, and levelSizes_.
Referenced by foreign_storage::anonymous_namespace{ForeignTableRefresh.cpp}::clear_cpu_and_gpu_cache(), and UpdelRoll::updateFragmenterAndCleanupChunks().
void Data_Namespace::DataMgr::deleteChunksWithPrefix | ( | const ChunkKey & | keyPrefix, |
const MemoryLevel | memLevel | ||
) |
Definition at line 504 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, and levelSizes_.
std::string Data_Namespace::DataMgr::dumpLevel | ( | const MemoryLevel | memLevel | ) |
Definition at line 418 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, cudaMgr_, and Data_Namespace::GPU_LEVEL.
void Data_Namespace::DataMgr::free | ( | AbstractBuffer * | buffer | ) |
Definition at line 525 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, Data_Namespace::AbstractBuffer::getDeviceId(), and Data_Namespace::AbstractBuffer::getType().
Referenced by UpdelRoll::cancelUpdate(), ThrustAllocator::deallocate(), CudaAllocator::free(), CudaAllocator::freeGpuAbstractBuffer(), CudaAllocator::~CudaAllocator(), InValuesBitmap::~InValuesBitmap(), PerfectHashTable::~PerfectHashTable(), StringDictionaryTranslationMgr::~StringDictionaryTranslationMgr(), and ThrustAllocator::~ThrustAllocator().
AbstractBuffer * Data_Namespace::DataMgr::getChunkBuffer | ( | const ChunkKey & | key, |
const MemoryLevel | memoryLevel, | ||
const int | deviceId = 0 , |
||
const size_t | numBytes = 0 |
||
) |
Definition at line 481 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK_LT, and levelSizes_.
Referenced by Chunk_NS::Chunk::getChunkBuffer().
const std::map<ChunkKey, File_Namespace::FileBuffer*>& Data_Namespace::DataMgr::getChunkMap | ( | ) |
void Data_Namespace::DataMgr::getChunkMetadataVecForKeyPrefix | ( | ChunkMetadataVector & | chunkMetadataVec, |
const ChunkKey & | keyPrefix | ||
) |
Definition at line 466 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by TableOptimizer::vacuumFragments().
Buffer_Namespace::CpuBufferMgr * Data_Namespace::DataMgr::getCpuBufferMgr | ( | ) | const |
Definition at line 638 of file DataMgr.cpp.
References bufferMgrs_, and Data_Namespace::CPU_LEVEL.
|
inline |
Definition at line 224 of file DataMgr.h.
References cudaMgr_.
Referenced by Executor::blockSize(), copy_to_nvidia_gpu(), CudaAllocator::copyFromDevice(), CudaAllocator::copyToDevice(), CudaAllocator::CudaAllocator(), Executor::cudaMgr(), get_available_gpus(), Executor::gridSize(), Executor::isCPUOnly(), CudaAllocator::setDeviceMem(), and CudaAllocator::zeroDeviceMem().
std::shared_ptr< ForeignStorageInterface > Data_Namespace::DataMgr::getForeignStorageInterface | ( | ) | const |
Definition at line 614 of file DataMgr.cpp.
References bufferMgrs_.
File_Namespace::GlobalFileMgr * Data_Namespace::DataMgr::getGlobalFileMgr | ( | ) | const |
Definition at line 606 of file DataMgr.cpp.
References bufferMgrs_, and CHECK.
Referenced by convertDB(), createTopLevelMetadata(), TableArchiver::dumpTable(), anonymous_namespace{DdlCommandExecutor.cpp}::get_agg_storage_stats(), getTableEpoch(), foreign_storage::InternalStorageStatsDataWrapper::initializeObjectsForTable(), resetTableEpochFloor(), TableArchiver::restoreTable(), setTableEpoch(), and TableOptimizer::vacuumDeletedRows().
Buffer_Namespace::GpuCudaBufferMgr * Data_Namespace::DataMgr::getGpuBufferMgr | ( | int32_t | device_id | ) | const |
Definition at line 643 of file DataMgr.cpp.
References bufferMgrs_, CHECK_GT, and Data_Namespace::GPU_LEVEL.
std::vector< MemoryInfo > Data_Namespace::DataMgr::getMemoryInfo | ( | const MemoryLevel | memLevel | ) | const |
Definition at line 349 of file DataMgr.cpp.
References buffer_access_mutex_, and getMemoryInfoUnlocked().
Referenced by Executor::createKernels().
std::vector< MemoryInfo > Data_Namespace::DataMgr::getMemoryInfoUnlocked | ( | const MemoryLevel | memLevel | ) | const |
Definition at line 354 of file DataMgr.cpp.
References bufferMgrs_, CHECK, Data_Namespace::MemoryData::chunk_key, Data_Namespace::CPU_LEVEL, cudaMgr_, Buffer_Namespace::BufferMgr::getAllocated(), Buffer_Namespace::BufferMgr::getMaxSize(), Buffer_Namespace::BufferMgr::getPageSize(), Buffer_Namespace::BufferMgr::getSlabSegments(), Data_Namespace::GPU_LEVEL, hasGpus_, Data_Namespace::MemoryInfo::isAllocationCapped, Buffer_Namespace::BufferMgr::isAllocationCapped(), Data_Namespace::MemoryInfo::maxNumPages, Data_Namespace::MemoryData::memStatus, Data_Namespace::MemoryInfo::nodeMemoryData, Data_Namespace::MemoryInfo::numPageAllocated, Data_Namespace::MemoryData::numPages, Data_Namespace::MemoryInfo::pageSize, Data_Namespace::MemoryData::slabNum, Data_Namespace::MemoryData::startPage, and Data_Namespace::MemoryData::touch.
Referenced by getMemoryInfo().
PersistentStorageMgr * Data_Namespace::DataMgr::getPersistentStorageMgr | ( | ) | const |
Definition at line 634 of file DataMgr.cpp.
References bufferMgrs_, and Data_Namespace::DISK_LEVEL.
Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::clear_cached_table_data(), and anonymous_namespace{RelAlgExecutor.cpp}::set_parallelism_hints().
DataMgr::SystemMemoryUsage Data_Namespace::DataMgr::getSystemMemoryUsage | ( | ) | const |
Definition at line 92 of file DataMgr.cpp.
References Data_Namespace::DataMgr::SystemMemoryUsage::frag, Data_Namespace::DataMgr::SystemMemoryUsage::free, Data_Namespace::ProcBuddyinfoParser::getFragmentationPercent(), Data_Namespace::DataMgr::SystemMemoryUsage::regular, Data_Namespace::DataMgr::SystemMemoryUsage::resident, Data_Namespace::DataMgr::SystemMemoryUsage::shared, Data_Namespace::DataMgr::SystemMemoryUsage::total, and Data_Namespace::DataMgr::SystemMemoryUsage::vtotal.
size_t Data_Namespace::DataMgr::getTableEpoch | ( | const int | db_id, |
const int | tb_id | ||
) |
Definition at line 592 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
|
static |
Definition at line 149 of file DataMgr.cpp.
Referenced by populateMgrs().
|
inline |
Definition at line 218 of file DataMgr.h.
References hasGpus_.
Referenced by get_available_gpus().
bool Data_Namespace::DataMgr::isBufferOnDevice | ( | const ChunkKey & | key, |
const MemoryLevel | memLevel, | ||
const int | deviceId | ||
) |
Definition at line 459 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by Chunk_NS::Chunk::isChunkOnDevice().
|
private |
Definition at line 216 of file DataMgr.cpp.
References allocateCpuBufferMgr(), bufferMgrs_, SystemParameters::cpu_buffer_mem_bytes, cudaMgr_, dataDir_, Data_Namespace::DRAM, g_pmem_size, getTotalSystemMemory(), SystemParameters::gpu_buffer_mem_bytes, hasGpus_, logger::INFO, levelSizes_, LOG, SystemParameters::max_cpu_slab_size, SystemParameters::max_gpu_slab_size, SystemParameters::min_cpu_slab_size, SystemParameters::min_gpu_slab_size, Data_Namespace::numCpuTiers, Data_Namespace::PMEM, reservedGpuMem_, and VLOG.
Referenced by resetPersistentStorage().
void Data_Namespace::DataMgr::removeTableRelatedDS | ( | const int | db_id, |
const int | tb_id | ||
) |
Definition at line 580 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
void Data_Namespace::DataMgr::resetPersistentStorage | ( | const File_Namespace::DiskCacheConfig & | cache_config, |
const size_t | num_reader_threads, | ||
const SystemParameters & | sys_params | ||
) |
Definition at line 202 of file DataMgr.cpp.
References bufferMgrs_, createTopLevelMetadata(), and populateMgrs().
void Data_Namespace::DataMgr::resetTableEpochFloor | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 599 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
void Data_Namespace::DataMgr::setTableEpoch | ( | const int | db_id, |
const int | tb_id, | ||
const int | start_epoch | ||
) |
Definition at line 585 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
|
mutableprivate |
Definition at line 277 of file DataMgr.h.
Referenced by alloc(), checkpoint(), clearMemory(), createChunkBuffer(), deleteChunksWithPrefix(), dumpLevel(), free(), getChunkBuffer(), getChunkMetadataVecForKeyPrefix(), getMemoryInfo(), isBufferOnDevice(), and removeTableRelatedDS().
|
private |
Definition at line 272 of file DataMgr.h.
Referenced by alloc(), allocateCpuBufferMgr(), checkpoint(), clearMemory(), convertDB(), createChunkBuffer(), createTopLevelMetadata(), deleteChunksWithPrefix(), dumpLevel(), free(), getChunkBuffer(), getChunkMetadataVecForKeyPrefix(), getCpuBufferMgr(), getForeignStorageInterface(), getGlobalFileMgr(), getGpuBufferMgr(), getMemoryInfoUnlocked(), getPersistentStorageMgr(), getTableEpoch(), isBufferOnDevice(), populateMgrs(), removeTableRelatedDS(), resetPersistentStorage(), resetTableEpochFloor(), setTableEpoch(), and ~DataMgr().
|
private |
Definition at line 273 of file DataMgr.h.
Referenced by allocateCpuBufferMgr(), clearMemory(), dumpLevel(), getCudaMgr(), getMemoryInfoUnlocked(), and populateMgrs().
|
private |
Definition at line 274 of file DataMgr.h.
Referenced by populateMgrs().
|
private |
Definition at line 275 of file DataMgr.h.
Referenced by getMemoryInfoUnlocked(), gpusPresent(), and populateMgrs().
std::vector<int> Data_Namespace::DataMgr::levelSizes_ |
Definition at line 229 of file DataMgr.h.
Referenced by alloc(), checkpoint(), deleteChunksWithPrefix(), getChunkBuffer(), and populateMgrs().
|
private |
Definition at line 276 of file DataMgr.h.
Referenced by populateMgrs().