OmniSciDB
085a039ca4
|
#include <DataMgr.h>
Classes | |
struct | SystemMemoryUsage |
Public Member Functions | |
DataMgr (const std::string &dataDir, const SystemParameters &system_parameters, std::unique_ptr< CudaMgr_Namespace::CudaMgr > cudaMgr, const bool useGpus, const size_t reservedGpuMem=(1<< 27), const size_t numReaderThreads=0, const File_Namespace::DiskCacheConfig cacheConfig=File_Namespace::DiskCacheConfig()) | |
~DataMgr () | |
AbstractBuffer * | createChunkBuffer (const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t page_size=0) |
AbstractBuffer * | getChunkBuffer (const ChunkKey &key, const MemoryLevel memoryLevel, const int deviceId=0, const size_t numBytes=0) |
void | deleteChunksWithPrefix (const ChunkKey &keyPrefix) |
void | deleteChunksWithPrefix (const ChunkKey &keyPrefix, const MemoryLevel memLevel) |
AbstractBuffer * | alloc (const MemoryLevel memoryLevel, const int deviceId, const size_t numBytes) |
void | free (AbstractBuffer *buffer) |
void | copy (AbstractBuffer *destBuffer, AbstractBuffer *srcBuffer) |
bool | isBufferOnDevice (const ChunkKey &key, const MemoryLevel memLevel, const int deviceId) |
std::vector< MemoryInfo > | getMemoryInfo (const MemoryLevel memLevel) const |
std::vector< MemoryInfo > | getMemoryInfoUnlocked (const MemoryLevel memLevel) const |
std::string | dumpLevel (const MemoryLevel memLevel) |
void | clearMemory (const MemoryLevel memLevel) |
const std::map< ChunkKey, File_Namespace::FileBuffer * > & | getChunkMap () |
void | checkpoint (const int db_id, const int tb_id) |
void | checkpoint (const int db_id, const int table_id, const MemoryLevel memory_level) |
void | getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) |
bool | gpusPresent () const |
void | removeTableRelatedDS (const int db_id, const int tb_id) |
void | setTableEpoch (const int db_id, const int tb_id, const int start_epoch) |
size_t | getTableEpoch (const int db_id, const int tb_id) |
void | resetTableEpochFloor (const int32_t db_id, const int32_t tb_id) |
CudaMgr_Namespace::CudaMgr * | getCudaMgr () const |
File_Namespace::GlobalFileMgr * | getGlobalFileMgr () const |
std::shared_ptr < ForeignStorageInterface > | getForeignStorageInterface () const |
SystemMemoryUsage | getSystemMemoryUsage () const |
PersistentStorageMgr * | getPersistentStorageMgr () const |
void | resetPersistentStorage (const File_Namespace::DiskCacheConfig &cache_config, const size_t num_reader_threads, const SystemParameters &sys_params) |
Buffer_Namespace::CpuBufferMgr * | getCpuBufferMgr () const |
Buffer_Namespace::GpuCudaBufferMgr * | getGpuBufferMgr (int32_t device_id) const |
Static Public Member Functions | |
static size_t | getTotalSystemMemory () |
Public Attributes | |
std::vector< int > | levelSizes_ |
Private Member Functions | |
void | populateMgrs (const SystemParameters &system_parameters, const size_t userSpecifiedNumReaderThreads, const File_Namespace::DiskCacheConfig &cache_config) |
void | convertDB (const std::string basePath) |
void | checkpoint () |
void | createTopLevelMetadata () const |
void | allocateCpuBufferMgr (int32_t device_id, size_t total_cpu_size, size_t minCpuSlabSize, size_t maxCpuSlabSize, size_t page_size, const std::vector< size_t > &cpu_tier_sizes) |
Private Attributes | |
std::vector< std::vector < AbstractBufferMgr * > > | bufferMgrs_ |
std::unique_ptr < CudaMgr_Namespace::CudaMgr > | cudaMgr_ |
std::string | dataDir_ |
bool | hasGpus_ |
size_t | reservedGpuMem_ |
std::mutex | buffer_access_mutex_ |
Friends | |
class | GlobalFileMgr |
|
explicit |
Definition at line 51 of file DataMgr.cpp.
Data_Namespace::DataMgr::~DataMgr | ( | ) |
Definition at line 83 of file DataMgr.cpp.
References bufferMgrs_.
AbstractBuffer * Data_Namespace::DataMgr::alloc | ( | const MemoryLevel | memoryLevel, |
const int | deviceId, | ||
const size_t | numBytes | ||
) |
Definition at line 519 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK_LT, and levelSizes_.
Referenced by ThrustAllocator::allocate(), ThrustAllocator::allocateScopedBuffer(), CudaAllocator::allocGpuAbstractBuffer(), InValuesBitmap::InValuesBitmap(), ColumnFetcher::linearizeFixedLenArrayColFrags(), and ColumnFetcher::linearizeVarLenArrayColFrags().
|
private |
Definition at line 172 of file DataMgr.cpp.
References bufferMgrs_, and cudaMgr_.
Referenced by populateMgrs().
void Data_Namespace::DataMgr::checkpoint | ( | const int | db_id, |
const int | tb_id | ||
) |
Definition at line 548 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by UpdelRoll::stageUpdate().
void Data_Namespace::DataMgr::checkpoint | ( | const int | db_id, |
const int | table_id, | ||
const MemoryLevel | memory_level | ||
) |
Definition at line 560 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK_LT, and levelSizes_.
|
private |
Definition at line 571 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by convertDB().
void Data_Namespace::DataMgr::clearMemory | ( | const MemoryLevel | memLevel | ) |
Definition at line 437 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK, cudaMgr_, Data_Namespace::GPU_LEVEL, logger::INFO, LOG, and logger::WARNING.
Referenced by Executor::clearMemory().
|
private |
Definition at line 303 of file DataMgr.cpp.
References bufferMgrs_, CHECK, checkpoint(), logger::FATAL, getGlobalFileMgr(), logger::INFO, shared::kDataDirectoryName, and LOG.
void Data_Namespace::DataMgr::copy | ( | AbstractBuffer * | destBuffer, |
AbstractBuffer * | srcBuffer | ||
) |
Definition at line 534 of file DataMgr.cpp.
References Data_Namespace::AbstractBuffer::getDeviceId(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::getType(), Data_Namespace::AbstractBuffer::size(), and Data_Namespace::AbstractBuffer::write().
AbstractBuffer * Data_Namespace::DataMgr::createChunkBuffer | ( | const ChunkKey & | key, |
const MemoryLevel | memoryLevel, | ||
const int | deviceId = 0 , |
||
const size_t | page_size = 0 |
||
) |
Definition at line 475 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by Chunk_NS::Chunk::createChunkBuffer().
|
private |
Definition at line 336 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
Referenced by resetPersistentStorage().
void Data_Namespace::DataMgr::deleteChunksWithPrefix | ( | const ChunkKey & | keyPrefix | ) |
Definition at line 495 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, and levelSizes_.
Referenced by anonymous_namespace{RelAlgExecutor.cpp}::prepare_for_system_table_execution(), UpdelRoll::updateFragmenterAndCleanupChunks(), StorageIOFacility::yieldDeleteCallback(), and StorageIOFacility::yieldUpdateCallback().
void Data_Namespace::DataMgr::deleteChunksWithPrefix | ( | const ChunkKey & | keyPrefix, |
const MemoryLevel | memLevel | ||
) |
Definition at line 507 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, and levelSizes_.
std::string Data_Namespace::DataMgr::dumpLevel | ( | const MemoryLevel | memLevel | ) |
Definition at line 421 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, cudaMgr_, and Data_Namespace::GPU_LEVEL.
void Data_Namespace::DataMgr::free | ( | AbstractBuffer * | buffer | ) |
Definition at line 528 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, Data_Namespace::AbstractBuffer::getDeviceId(), and Data_Namespace::AbstractBuffer::getType().
Referenced by UpdelRoll::cancelUpdate(), ThrustAllocator::deallocate(), CudaAllocator::free(), CudaAllocator::freeGpuAbstractBuffer(), CudaAllocator::~CudaAllocator(), InValuesBitmap::~InValuesBitmap(), PerfectHashTable::~PerfectHashTable(), StringDictionaryTranslationMgr::~StringDictionaryTranslationMgr(), and ThrustAllocator::~ThrustAllocator().
AbstractBuffer * Data_Namespace::DataMgr::getChunkBuffer | ( | const ChunkKey & | key, |
const MemoryLevel | memoryLevel, | ||
const int | deviceId = 0 , |
||
const size_t | numBytes = 0 |
||
) |
Definition at line 484 of file DataMgr.cpp.
References buffer_access_mutex_, bufferMgrs_, CHECK_LT, and levelSizes_.
Referenced by Chunk_NS::Chunk::getChunkBuffer().
const std::map<ChunkKey, File_Namespace::FileBuffer*>& Data_Namespace::DataMgr::getChunkMap | ( | ) |
void Data_Namespace::DataMgr::getChunkMetadataVecForKeyPrefix | ( | ChunkMetadataVector & | chunkMetadataVec, |
const ChunkKey & | keyPrefix | ||
) |
Definition at line 469 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by TableOptimizer::vacuumFragments().
Buffer_Namespace::CpuBufferMgr * Data_Namespace::DataMgr::getCpuBufferMgr | ( | ) | const |
Definition at line 641 of file DataMgr.cpp.
References bufferMgrs_, and Data_Namespace::CPU_LEVEL.
|
inline |
Definition at line 222 of file DataMgr.h.
References cudaMgr_.
Referenced by Executor::blockSize(), copy_to_nvidia_gpu(), CudaAllocator::copyFromDevice(), CudaAllocator::copyToDevice(), CudaAllocator::CudaAllocator(), Executor::cudaMgr(), get_available_gpus(), Executor::gridSize(), Executor::isCPUOnly(), CudaAllocator::setDeviceMem(), and CudaAllocator::zeroDeviceMem().
std::shared_ptr< ForeignStorageInterface > Data_Namespace::DataMgr::getForeignStorageInterface | ( | ) | const |
Definition at line 617 of file DataMgr.cpp.
References bufferMgrs_.
File_Namespace::GlobalFileMgr * Data_Namespace::DataMgr::getGlobalFileMgr | ( | ) | const |
Definition at line 609 of file DataMgr.cpp.
References bufferMgrs_, and CHECK.
Referenced by convertDB(), createTopLevelMetadata(), TableArchiver::dumpTable(), anonymous_namespace{DdlCommandExecutor.cpp}::get_agg_storage_stats(), getTableEpoch(), foreign_storage::InternalStorageStatsDataWrapper::initializeObjectsForTable(), resetTableEpochFloor(), TableArchiver::restoreTable(), setTableEpoch(), and TableOptimizer::vacuumDeletedRows().
Buffer_Namespace::GpuCudaBufferMgr * Data_Namespace::DataMgr::getGpuBufferMgr | ( | int32_t | device_id | ) | const |
Definition at line 646 of file DataMgr.cpp.
References bufferMgrs_, CHECK_GT, and Data_Namespace::GPU_LEVEL.
std::vector< MemoryInfo > Data_Namespace::DataMgr::getMemoryInfo | ( | const MemoryLevel | memLevel | ) | const |
Definition at line 352 of file DataMgr.cpp.
References buffer_access_mutex_, and getMemoryInfoUnlocked().
Referenced by Executor::createKernels().
std::vector< MemoryInfo > Data_Namespace::DataMgr::getMemoryInfoUnlocked | ( | const MemoryLevel | memLevel | ) | const |
Definition at line 357 of file DataMgr.cpp.
References bufferMgrs_, CHECK, Data_Namespace::MemoryData::chunk_key, Data_Namespace::CPU_LEVEL, cudaMgr_, Buffer_Namespace::BufferMgr::getAllocated(), Buffer_Namespace::BufferMgr::getMaxSize(), Buffer_Namespace::BufferMgr::getPageSize(), Buffer_Namespace::BufferMgr::getSlabSegments(), Data_Namespace::GPU_LEVEL, hasGpus_, Data_Namespace::MemoryInfo::isAllocationCapped, Buffer_Namespace::BufferMgr::isAllocationCapped(), Data_Namespace::MemoryInfo::maxNumPages, Data_Namespace::MemoryData::memStatus, Data_Namespace::MemoryInfo::nodeMemoryData, Data_Namespace::MemoryInfo::numPageAllocated, Data_Namespace::MemoryData::numPages, Data_Namespace::MemoryInfo::pageSize, Data_Namespace::MemoryData::slabNum, Data_Namespace::MemoryData::startPage, and Data_Namespace::MemoryData::touch.
Referenced by getMemoryInfo().
PersistentStorageMgr * Data_Namespace::DataMgr::getPersistentStorageMgr | ( | ) | const |
Definition at line 637 of file DataMgr.cpp.
References bufferMgrs_, and Data_Namespace::DISK_LEVEL.
Referenced by RelAlgExecutor::initializeParallelismHints(), and anonymous_namespace{RelAlgExecutor.cpp}::set_parallelism_hints().
DataMgr::SystemMemoryUsage Data_Namespace::DataMgr::getSystemMemoryUsage | ( | ) | const |
Definition at line 92 of file DataMgr.cpp.
References Data_Namespace::DataMgr::SystemMemoryUsage::frag, Data_Namespace::DataMgr::SystemMemoryUsage::free, Data_Namespace::ProcBuddyinfoParser::getFragmentationPercent(), Data_Namespace::DataMgr::SystemMemoryUsage::regular, Data_Namespace::DataMgr::SystemMemoryUsage::resident, Data_Namespace::DataMgr::SystemMemoryUsage::shared, Data_Namespace::DataMgr::SystemMemoryUsage::total, and Data_Namespace::DataMgr::SystemMemoryUsage::vtotal.
size_t Data_Namespace::DataMgr::getTableEpoch | ( | const int | db_id, |
const int | tb_id | ||
) |
Definition at line 595 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
|
static |
Definition at line 149 of file DataMgr.cpp.
Referenced by populateMgrs().
|
inline |
Definition at line 216 of file DataMgr.h.
References hasGpus_.
Referenced by get_available_gpus().
bool Data_Namespace::DataMgr::isBufferOnDevice | ( | const ChunkKey & | key, |
const MemoryLevel | memLevel, | ||
const int | deviceId | ||
) |
Definition at line 462 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
Referenced by Chunk_NS::Chunk::isChunkOnDevice().
|
private |
Definition at line 216 of file DataMgr.cpp.
References allocateCpuBufferMgr(), bufferMgrs_, SystemParameters::cpu_buffer_mem_bytes, cudaMgr_, dataDir_, Data_Namespace::DRAM, g_pmem_size, getTotalSystemMemory(), SystemParameters::gpu_buffer_mem_bytes, hasGpus_, logger::INFO, levelSizes_, LOG, SystemParameters::max_cpu_slab_size, SystemParameters::max_gpu_slab_size, SystemParameters::min_cpu_slab_size, SystemParameters::min_gpu_slab_size, Data_Namespace::numCpuTiers, Data_Namespace::PMEM, reservedGpuMem_, and VLOG.
Referenced by resetPersistentStorage().
void Data_Namespace::DataMgr::removeTableRelatedDS | ( | const int | db_id, |
const int | tb_id | ||
) |
Definition at line 583 of file DataMgr.cpp.
References buffer_access_mutex_, and bufferMgrs_.
void Data_Namespace::DataMgr::resetPersistentStorage | ( | const File_Namespace::DiskCacheConfig & | cache_config, |
const size_t | num_reader_threads, | ||
const SystemParameters & | sys_params | ||
) |
Definition at line 202 of file DataMgr.cpp.
References bufferMgrs_, createTopLevelMetadata(), and populateMgrs().
void Data_Namespace::DataMgr::resetTableEpochFloor | ( | const int32_t | db_id, |
const int32_t | tb_id | ||
) |
Definition at line 602 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
void Data_Namespace::DataMgr::setTableEpoch | ( | const int | db_id, |
const int | tb_id, | ||
const int | start_epoch | ||
) |
Definition at line 588 of file DataMgr.cpp.
References bufferMgrs_, CHECK, and getGlobalFileMgr().
|
mutableprivate |
Definition at line 275 of file DataMgr.h.
Referenced by alloc(), checkpoint(), clearMemory(), createChunkBuffer(), deleteChunksWithPrefix(), dumpLevel(), free(), getChunkBuffer(), getChunkMetadataVecForKeyPrefix(), getMemoryInfo(), isBufferOnDevice(), and removeTableRelatedDS().
|
private |
Definition at line 270 of file DataMgr.h.
Referenced by alloc(), allocateCpuBufferMgr(), checkpoint(), clearMemory(), convertDB(), createChunkBuffer(), createTopLevelMetadata(), deleteChunksWithPrefix(), dumpLevel(), free(), getChunkBuffer(), getChunkMetadataVecForKeyPrefix(), getCpuBufferMgr(), getForeignStorageInterface(), getGlobalFileMgr(), getGpuBufferMgr(), getMemoryInfoUnlocked(), getPersistentStorageMgr(), getTableEpoch(), isBufferOnDevice(), populateMgrs(), removeTableRelatedDS(), resetPersistentStorage(), resetTableEpochFloor(), setTableEpoch(), and ~DataMgr().
|
private |
Definition at line 271 of file DataMgr.h.
Referenced by allocateCpuBufferMgr(), clearMemory(), dumpLevel(), getCudaMgr(), getMemoryInfoUnlocked(), and populateMgrs().
|
private |
Definition at line 272 of file DataMgr.h.
Referenced by populateMgrs().
|
private |
Definition at line 273 of file DataMgr.h.
Referenced by getMemoryInfoUnlocked(), gpusPresent(), and populateMgrs().
std::vector<int> Data_Namespace::DataMgr::levelSizes_ |
Definition at line 227 of file DataMgr.h.
Referenced by alloc(), checkpoint(), deleteChunksWithPrefix(), getChunkBuffer(), and populateMgrs().
|
private |
Definition at line 274 of file DataMgr.h.
Referenced by populateMgrs().