#include <ForeignStorageInterface.h>

Inheritance diagram for ForeignStorageBufferMgr:

Collaboration diagram for ForeignStorageBufferMgr:

Public Member Functions
	ForeignStorageBufferMgr (const int db_id, const int table_id, PersistentForeignStorageInterface *persistent_foreign_storage)

void	checkpoint () override

Data_Namespace::AbstractBuffer *	createBuffer (const ChunkKey &key, const size_t pageSize=0, const size_t initialSize=0) override

Data_Namespace::AbstractBuffer *	getBuffer (const ChunkKey &key, const size_t numBytes=0) override

void	fetchBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *destBuffer, const size_t numBytes=0) override

void	getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override

std::string	getStringMgrType () override

size_t	getNumChunks () override

void	deleteBuffer (const ChunkKey &key, const bool purge=true) override

void	deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override

Data_Namespace::AbstractBuffer *	putBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *srcBuffer, const size_t numBytes=0) override

bool	isBufferOnDevice (const ChunkKey &key) override

std::string	printSlabs () override

size_t	getMaxSize () override

size_t	getInUseSize () override

size_t	getAllocated () override

bool	isAllocationCapped () override

void	checkpoint (const int db_id, const int tb_id) override

Data_Namespace::AbstractBuffer *	alloc (const size_t numBytes=0) override

void	free (Data_Namespace::AbstractBuffer *buffer) override

MgrType	getMgrType () override

void	removeTableRelatedDS (const int db_id, const int table_id) override

Private Attributes
PersistentForeignStorageInterface *	persistent_foreign_storage_

std::map< ChunkKey, std::unique_ptr < ForeignStorageBuffer > >	chunk_index_

heavyai::shared_mutex	chunk_index_mutex_

Detailed Description

Definition at line 120 of file ForeignStorageInterface.h.

Constructor & Destructor Documentation

ForeignStorageBufferMgr::ForeignStorageBufferMgr	(	const int	db_id,
		const int	table_id,
		PersistentForeignStorageInterface *	persistent_foreign_storage
	)

Definition at line 51 of file ForeignStorageInterface.cpp.

55 : AbstractBufferMgr(0), persistent_foreign_storage_(persistent_foreign_storage) {}

ForeignStorageBufferMgr::persistent_foreign_storage_

PersistentForeignStorageInterface * persistent_foreign_storage_

Definition: ForeignStorageInterface.h:215

Member Function Documentation

Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::alloc ( const size_t numBytes = 0 )

inlineoverride

Definition at line 198 of file ForeignStorageInterface.h.

References CHECK.

                                                                           {
     CHECK(false);
     return nullptr;
   }

void ForeignStorageBufferMgr::checkpoint ( )

override

Definition at line 57 of file ForeignStorageInterface.cpp.

References PersistentForeignStorageInterface::append(), chunk_index_, and persistent_foreign_storage_.

                                          {
   // TODO(alex)
   std::vector<ForeignStorageColumnBuffer> column_buffers;
   for (auto& kv : chunk_index_) {
     const auto buffer = kv.second->moveBuffer();
     column_buffers.emplace_back(
         ForeignStorageColumnBuffer{kv.first, kv.second->getSqlType(), buffer});
   }
   persistent_foreign_storage_->append(column_buffers);
 }

Here is the call graph for this function:

void ForeignStorageBufferMgr::checkpoint	(	const int	db_id,
		const int	tb_id
	)

inlineoverride

Definition at line 195 of file ForeignStorageInterface.h.

References CHECK.

195 { CHECK(false); }

CHECK

#define CHECK(condition)

Definition: Logger.h:291

Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::createBuffer	(	const ChunkKey &	key,
		const size_t	pageSize = `0`,
		const size_t	initialSize = `0`
	)

override

Definition at line 68 of file ForeignStorageInterface.cpp.

References chunk_index_, chunk_index_mutex_, and persistent_foreign_storage_.

                               {
   heavyai::unique_lock<heavyai::shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
   const auto it_ok = chunk_index_.emplace(
       key, std::make_unique<ForeignStorageBuffer>(key, persistent_foreign_storage_));
   // this check fails if we create table, drop it and create again
   // CHECK(it_ok.second);
   return it_ok.first->second.get();
 }

void ForeignStorageBufferMgr::deleteBuffer	(	const ChunkKey &	key,
		const bool	purge = `true`
	)

inlineoverride

Definition at line 149 of file ForeignStorageInterface.h.

References CHECK.

                                                                            {
     CHECK(false);
   }

void ForeignStorageBufferMgr::deleteBuffersWithPrefix	(	const ChunkKey &	keyPrefix,
		const bool	purge = `true`
	)

inlineoverride

Definition at line 153 of file ForeignStorageInterface.h.

References CHECK.

                                                                  {
     CHECK(false);
   }

void ForeignStorageBufferMgr::fetchBuffer	(	const ChunkKey &	key,
		Data_Namespace::AbstractBuffer *	destBuffer,
		const size_t	numBytes = `0`
	)

override

Definition at line 89 of file ForeignStorageInterface.cpp.

References CHECK, getBuffer(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::setMemoryPtr(), Data_Namespace::AbstractBuffer::setSize(), and Data_Namespace::AbstractBuffer::syncEncoder().

                                                                  {
   CHECK(numBytes);
   auto file_buffer = dynamic_cast<ForeignStorageBuffer*>(getBuffer(key, numBytes));
   CHECK(file_buffer);
 
   // TODO: check if GPU is used
   auto buf = file_buffer->tryZeroCopy(numBytes);
   if (buf) {
     destBuffer->setMemoryPtr(buf);
   } else {
     destBuffer->reserve(numBytes);
     file_buffer->read(destBuffer->getMemoryPtr(), numBytes);
   }
   destBuffer->setSize(numBytes);
   destBuffer->syncEncoder(file_buffer);
 }

Here is the call graph for this function:

void ForeignStorageBufferMgr::free ( Data_Namespace::AbstractBuffer * buffer )

inlineoverride

Definition at line 203 of file ForeignStorageInterface.h.

References CHECK.

203 { CHECK(false); }

CHECK

#define CHECK(condition)

Definition: Logger.h:291

size_t ForeignStorageBufferMgr::getAllocated ( )

inlineoverride

Definition at line 185 of file ForeignStorageInterface.h.

References CHECK.

                                  {
     CHECK(false);
     return 0;
   }

Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::getBuffer	(	const ChunkKey &	key,
		const size_t	numBytes = `0`
	)

override

Definition at line 80 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, and chunk_index_mutex_.

Referenced by fetchBuffer().

                            {
   heavyai::shared_lock<heavyai::shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
   const auto it = chunk_index_.find(key);
   CHECK(it != chunk_index_.end());
   return it->second.get();
 }

Here is the caller graph for this function:

void ForeignStorageBufferMgr::getChunkMetadataVecForKeyPrefix	(	ChunkMetadataVector &	chunkMetadataVec,
		const ChunkKey &	keyPrefix
	)

override

Definition at line 108 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, chunk_index_mutex_, and run_benchmark_import::type.

                                {
   heavyai::unique_lock<heavyai::shared_mutex> chunk_index_write_lock(
       chunk_index_mutex_);  // is this guarding the right structure?  it look slike we
                             // oly read here for chunk
   auto chunk_it = chunk_index_.lower_bound(keyPrefix);
   if (chunk_it == chunk_index_.end()) {
     CHECK(false);  // throw?
   }
 
   while (chunk_it != chunk_index_.end() &&
          std::search(chunk_it->first.begin(),
                      chunk_it->first.begin() + keyPrefix.size(),
                      keyPrefix.begin(),
                      keyPrefix.end()) != chunk_it->first.begin() + keyPrefix.size()) {
     const auto& chunk_key = chunk_it->first;
     if (chunk_key.size() == 5) {
       if (chunk_key[4] == 1) {
         const auto& buffer = *chunk_it->second;
         auto type = buffer.getSqlType();
         auto size = buffer.size();
         auto subkey = chunk_key;
         subkey[4] = 2;
         auto& index_buf = *(chunk_index_.find(subkey)->second);
         auto bs = index_buf.size() / index_buf.getSqlType().get_size();
         auto chunk_metadata =
             std::make_shared<ChunkMetadata>(type, size, bs, ChunkStats{});
         chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
       }
     } else {
       const auto& buffer = *chunk_it->second;
       auto chunk_metadata = std::make_shared<ChunkMetadata>();
       chunk_metadata->sqlType = buffer.getSqlType();
       buffer.getEncoder()->getMetadata(chunk_metadata);
       chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
     }
     chunk_it++;
   }
 }

size_t ForeignStorageBufferMgr::getInUseSize ( )

inlineoverride

Definition at line 180 of file ForeignStorageInterface.h.

References CHECK.

                                  {
     CHECK(false);
     return 0;
   }

size_t ForeignStorageBufferMgr::getMaxSize ( )

inlineoverride

Definition at line 175 of file ForeignStorageInterface.h.

References CHECK.

                                {
     CHECK(false);
     return 0;
   }

MgrType ForeignStorageBufferMgr::getMgrType ( )

inlineoverride

Definition at line 205 of file ForeignStorageInterface.h.

References CHECK.

                                 {
     CHECK(false);
     return FILE_MGR;
   }

size_t ForeignStorageBufferMgr::getNumChunks ( )

inlineoverride

Definition at line 143 of file ForeignStorageInterface.h.

References chunk_index_, and chunk_index_mutex_.

                                  {
     heavyai::shared_lock<heavyai::shared_mutex> chunk_index_write_lock(
         chunk_index_mutex_);
     return chunk_index_.size();
   }

std::string ForeignStorageBufferMgr::getStringMgrType ( )

inlineoverride

Definition at line 141 of file ForeignStorageInterface.h.

141 { return ToString(FILE_MGR); }

bool ForeignStorageBufferMgr::isAllocationCapped ( )

inlineoverride

Definition at line 190 of file ForeignStorageInterface.h.

References CHECK.

                                      {
     CHECK(false);
     return false;
   }

bool ForeignStorageBufferMgr::isBufferOnDevice ( const ChunkKey & key )

inlineoverride

Definition at line 165 of file ForeignStorageInterface.h.

References CHECK.

                                                       {
     CHECK(false);
     return false;
   }

std::string ForeignStorageBufferMgr::printSlabs ( )

inlineoverride

Definition at line 170 of file ForeignStorageInterface.h.

References CHECK.

                                   {
     CHECK(false);
     return "";
   }

Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::putBuffer	(	const ChunkKey &	key,
		Data_Namespace::AbstractBuffer *	srcBuffer,
		const size_t	numBytes = `0`
	)

inlineoverride

Definition at line 158 of file ForeignStorageInterface.h.

References CHECK.

                                                                                 {
     CHECK(false);
     return nullptr;
   }

void ForeignStorageBufferMgr::removeTableRelatedDS	(	const int	db_id,
		const int	table_id
	)

inlineoverride

Definition at line 210 of file ForeignStorageInterface.h.

References PersistentForeignStorageInterface::dropTable(), and persistent_foreign_storage_.

                                                                           {
     persistent_foreign_storage_->dropTable(db_id, table_id);
   }

Here is the call graph for this function:

Member Data Documentation

std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer> > ForeignStorageBufferMgr::chunk_index_

private

Definition at line 216 of file ForeignStorageInterface.h.

Referenced by checkpoint(), createBuffer(), getBuffer(), getChunkMetadataVecForKeyPrefix(), and getNumChunks().

heavyai::shared_mutex ForeignStorageBufferMgr::chunk_index_mutex_

mutableprivate

Definition at line 217 of file ForeignStorageInterface.h.

Referenced by createBuffer(), getBuffer(), getChunkMetadataVecForKeyPrefix(), and getNumChunks().

PersistentForeignStorageInterface* ForeignStorageBufferMgr::persistent_foreign_storage_

private

Definition at line 215 of file ForeignStorageInterface.h.

Referenced by checkpoint(), createBuffer(), and removeTableRelatedDS().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageInterface.h
/home/jenkins-slave/workspace/core-os-doxygen/DataMgr/ForeignStorage/ForeignStorageInterface.cpp

Public Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation