Data_Namespace::AbstractBuffer Class Reference

An AbstractBuffer is a unit of data management for a data manager. More...

#include <AbstractBuffer.h>

Inheritance diagram for Data_Namespace::AbstractBuffer:

Collaboration diagram for Data_Namespace::AbstractBuffer:

Public Member Functions
	AbstractBuffer (const int device_id)
	AbstractBuffer (const int device_id, const SQLTypeInfo sql_type)
virtual	~AbstractBuffer ()
virtual void	read (int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0
virtual void	write (int8_t *src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int src_device_id=-1)=0
virtual void	reserve (size_t num_bytes)=0
virtual void	append (int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
virtual int8_t *	getMemoryPtr ()=0
virtual void	setMemoryPtr (int8_t *new_ptr)
virtual size_t	pageCount () const =0
virtual size_t	pageSize () const =0
virtual size_t	reservedSize () const =0
virtual MemoryLevel	getType () const =0
virtual int	pin ()
virtual int	unPin ()
virtual int	getPinCount ()
size_t	size () const
int	getDeviceId () const
bool	isDirty () const
bool	isAppended () const
bool	isUpdated () const
bool	hasEncoder () const
SQLTypeInfo	getSqlType () const
void	setSqlType (const SQLTypeInfo &sql_type)
Encoder *	getEncoder () const
void	setDirty ()
void	setUpdated ()
void	setAppended ()
void	setSize (const size_t size)
void	clearDirtyBits ()
void	initEncoder (const SQLTypeInfo &tmp_sql_type)
void	syncEncoder (const AbstractBuffer *src_buffer)
void	copyTo (AbstractBuffer *destination_buffer, const size_t num_bytes=0)
void	resetToEmpty ()

Protected Attributes
std::unique_ptr< Encoder >	encoder_
SQLTypeInfo	sql_type_
size_t	size_
int	device_id_

Private Attributes
bool	is_dirty_
bool	is_appended_
bool	is_updated_

Detailed Description

An AbstractBuffer is a unit of data management for a data manager.

Definition at line 47 of file AbstractBuffer.h.

Constructor & Destructor Documentation

Data_Namespace::AbstractBuffer::AbstractBuffer ( const int  device_id )

inline

Definition at line 49 of file AbstractBuffer.h.

      : encoder_(nullptr)
      , size_(0)
      , device_id_(device_id)
      , is_dirty_(false)
      , is_appended_(false)
      , is_updated_(false) {}

Data_Namespace::AbstractBuffer::AbstractBuffer ( const int  device_id, const SQLTypeInfo  sql_type  )

inline

Definition at line 57 of file AbstractBuffer.h.

References initEncoder().

      : size_(0)
      , device_id_(device_id)
      , is_dirty_(false)
      , is_appended_(false)
      , is_updated_(false) {
    initEncoder(sql_type);
  }

Here is the call graph for this function:

virtual Data_Namespace::AbstractBuffer::~AbstractBuffer ( )

inlinevirtual

Definition at line 65 of file AbstractBuffer.h.

{}

Member Function Documentation

virtual void Data_Namespace::AbstractBuffer::append ( int8_t *  src, const size_t  num_bytes, const MemoryLevel  src_buffer_type = CPU_LEVEL, const int  device_id = -1  )

pure virtual

Implemented in Buffer_Namespace::Buffer, ForeignStorageBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendArraysToBuffer(), foreign_storage::ParquetArrayEncoder::appendArraysToBuffer(), foreign_storage::ParquetStringNoneEncoder::appendData(), DateDaysEncoder< T, V >::appendData(), FixedLengthEncoder< T, V >::appendData(), NoneEncoder< T >::appendData(), foreign_storage::ParquetInPlaceEncoder::appendData(), StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t >::appendData(), Buffer_Namespace::BufferMgr::putBuffer(), and foreign_storage::ParquetStringNoneEncoder::writeInitialOffsetIfApplicable().

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::clearDirtyBits ( )

inline

Definition at line 121 of file AbstractBuffer.h.

References is_appended_, is_dirty_, and is_updated_.

Referenced by Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

                               {
    is_appended_ = false;
    is_updated_ = false;
    is_dirty_ = false;
  }

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::copyTo	(	AbstractBuffer *	destination_buffer,
		const size_t	num_bytes = 0
	)

Definition at line 44 of file AbstractBuffer.cpp.

References getDeviceId(), getMemoryPtr(), getType(), isUpdated(), read(), reserve(), setSize(), size(), and syncEncoder().

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), PersistentStorageMgr::fetchBuffer(), Buffer_Namespace::BufferMgr::fetchBuffer(), and File_Namespace::FileMgr::fetchBuffer().

                                                                                      {
  size_t chunk_size = (num_bytes == 0) ? size() : num_bytes;
  destination_buffer->reserve(chunk_size);
  if (isUpdated()) {
    read(destination_buffer->getMemoryPtr(),
         chunk_size,
         0,
         destination_buffer->getType(),
         destination_buffer->getDeviceId());
  } else {
    read(destination_buffer->getMemoryPtr() + destination_buffer->size(),
         chunk_size - destination_buffer->size(),
         destination_buffer->size(),
         destination_buffer->getType(),
         destination_buffer->getDeviceId());
  }
  destination_buffer->setSize(chunk_size);
  destination_buffer->syncEncoder(this);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int Data_Namespace::AbstractBuffer::getDeviceId ( ) const

inline

Definition at line 96 of file AbstractBuffer.h.

References device_id_.

Referenced by Data_Namespace::DataMgr::copy(), copyTo(), Data_Namespace::DataMgr::free(), Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

{ return device_id_; }

Here is the caller graph for this function:

Encoder* Data_Namespace::AbstractBuffer::getEncoder ( ) const

inline

Definition at line 103 of file AbstractBuffer.h.

References CHECK, encoder_, and hasEncoder().

Referenced by Chunk_NS::Chunk::appendData(), foreign_storage::ForeignStorageCache::cacheMetadataVec(), Chunk_NS::Chunk::getChunkBuffer(), Chunk_NS::Chunk::getNumElemsForBytesInsertData(), foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t >::getUpdatedStats(), Chunk_NS::Chunk::initEncoder(), foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::reduce_metadata(), and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer().

                                     {
    CHECK(hasEncoder());
    return encoder_.get();
  }

Here is the call graph for this function:

Here is the caller graph for this function:

virtual int8_t* Data_Namespace::AbstractBuffer::getMemoryPtr ( )

pure virtual

Implemented in File_Namespace::FileBuffer, Buffer_Namespace::Buffer, ForeignStorageBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by ThrustAllocator::allocate(), ThrustAllocator::allocateScopedBuffer(), Fragmenter_Namespace::ArrayChunkConverter::ArrayChunkConverter(), Chunk_NS::Chunk::begin_iterator(), Data_Namespace::DataMgr::copy(), copyTo(), Fragmenter_Namespace::DateChunkConverter< BUFFER_DATA_TYPE >::DateChunkConverter(), ForeignStorageBufferMgr::fetchBuffer(), Fragmenter_Namespace::FixedLenArrayChunkConverter::FixedLenArrayChunkConverter(), PerfectHashTable::getGpuBuffer(), Buffer_Namespace::BufferMgr::putBuffer(), File_Namespace::FileMgr::putBuffer(), Fragmenter_Namespace::ScalarChunkConverter< BUFFER_DATA_TYPE, INSERT_DATA_TYPE >::ScalarChunkConverter(), and Fragmenter_Namespace::StringChunkConverter::StringChunkConverter().

Here is the caller graph for this function:

virtual int Data_Namespace::AbstractBuffer::getPinCount ( )

inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 92 of file AbstractBuffer.h.

Referenced by ThrustAllocator::allocate(), and ThrustAllocator::allocateScopedBuffer().

{ return 0; }

Here is the caller graph for this function:

SQLTypeInfo Data_Namespace::AbstractBuffer::getSqlType ( ) const

inline

Definition at line 101 of file AbstractBuffer.h.

References sql_type_.

Referenced by Encoder::getMetadata(), FixedLengthArrayNoneEncoder::is_null(), FixedLengthArrayNoneEncoder::resetChunkStats(), ArrayNoneEncoder::resetChunkStats(), FixedLengthArrayNoneEncoder::update_elem_stats(), and ArrayNoneEncoder::update_elem_stats().

{ return sql_type_; }

Here is the caller graph for this function:

virtual MemoryLevel Data_Namespace::AbstractBuffer::getType ( ) const

pure virtual

Implemented in File_Namespace::FileBuffer, ForeignStorageBuffer, foreign_storage::ForeignStorageBuffer, Buffer_Namespace::CpuBuffer, and Buffer_Namespace::GpuCudaBuffer.

Referenced by RowSetMemoryOwner::addVarlenInputBuffer(), Data_Namespace::DataMgr::copy(), copyTo(), Data_Namespace::DataMgr::free(), Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

Here is the caller graph for this function:

bool Data_Namespace::AbstractBuffer::hasEncoder ( ) const

inline

Definition at line 100 of file AbstractBuffer.h.

References encoder_.

Referenced by getEncoder(), syncEncoder(), and File_Namespace::FileBuffer::writeMetadata().

{ return (encoder_ != nullptr); }

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::initEncoder

(

const SQLTypeInfo &

tmp_sql_type

)

Definition at line 26 of file AbstractBuffer.cpp.

References Encoder::Create(), encoder_, logger::FATAL, SQLTypeInfo::get_type_name(), LOG_IF, and sql_type_.

Referenced by AbstractBuffer(), foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::add_placeholder_metadata(), foreign_storage::ParquetEncoder::createMetadata(), foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t >::getUpdatedStats(), Chunk_NS::Chunk::initEncoder(), File_Namespace::FileBuffer::readMetadata(), foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::reduce_metadata(), foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and syncEncoder().

                                                                {
  sql_type_ = tmp_sql_type;
  encoder_.reset(Encoder::Create(this, sql_type_));
  LOG_IF(FATAL, encoder_ == nullptr)
      << "Failed to create encoder for SQL Type " << sql_type_.get_type_name();
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool Data_Namespace::AbstractBuffer::isAppended ( ) const

inline

Definition at line 98 of file AbstractBuffer.h.

References is_appended_.

Referenced by Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

{ return is_appended_; }

Here is the caller graph for this function:

bool Data_Namespace::AbstractBuffer::isDirty ( ) const

inline

Definition at line 97 of file AbstractBuffer.h.

References is_dirty_.

Referenced by StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), foreign_storage::ForeignStorageCache::cacheChunk(), foreign_storage::CachingForeignStorageMgr::fetchBuffer(), File_Namespace::FileMgr::fetchBuffer(), Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

{ return is_dirty_; }

Here is the caller graph for this function:

bool Data_Namespace::AbstractBuffer::isUpdated ( ) const

inline

Definition at line 99 of file AbstractBuffer.h.

References is_updated_.

Referenced by copyTo(), Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

{ return is_updated_; }

Here is the caller graph for this function:

virtual size_t Data_Namespace::AbstractBuffer::pageCount ( ) const

pure virtual

Implemented in File_Namespace::FileBuffer, Buffer_Namespace::Buffer, ForeignStorageBuffer, and foreign_storage::ForeignStorageBuffer.

virtual size_t Data_Namespace::AbstractBuffer::pageSize ( ) const

pure virtual

Implemented in File_Namespace::FileBuffer, Buffer_Namespace::Buffer, ForeignStorageBuffer, and foreign_storage::ForeignStorageBuffer.

virtual int Data_Namespace::AbstractBuffer::pin ( )

inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 90 of file AbstractBuffer.h.

Referenced by Buffer_Namespace::BufferMgr::fetchBuffer().

{ return 0; }

Here is the caller graph for this function:

virtual void Data_Namespace::AbstractBuffer::read ( int8_t *const  dst, const size_t  num_bytes, const size_t  offset = 0, const MemoryLevel  dst_buffer_type = CPU_LEVEL, const int  dst_device_id = -1  )

pure virtual

Implemented in Buffer_Namespace::Buffer, ForeignStorageBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by StringNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), and copyTo().

Here is the caller graph for this function:

virtual void Data_Namespace::AbstractBuffer::reserve ( size_t  num_bytes )

pure virtual

Implemented in Buffer_Namespace::Buffer, ForeignStorageBuffer, File_Namespace::FileBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by foreign_storage::ParquetStringNoneEncoder::appendData(), StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), copyTo(), ForeignStorageBufferMgr::fetchBuffer(), and foreign_storage::CsvDataWrapper::populateChunkMapForColumns().

Here is the caller graph for this function:

virtual size_t Data_Namespace::AbstractBuffer::reservedSize ( ) const

pure virtual

Implemented in File_Namespace::FileBuffer, Buffer_Namespace::Buffer, ForeignStorageBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by PerfectHashTable::getHashTableBufferSize(), and PerfectHashTable::gpuReservedSize().

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::resetToEmpty

(

)

Definition at line 64 of file AbstractBuffer.cpp.

References encoder_, and size_.

                                  {
  encoder_ = nullptr;
  size_ = 0;
}

void Data_Namespace::AbstractBuffer::setAppended ( )

inline

Definition at line 115 of file AbstractBuffer.h.

References is_appended_, and is_dirty_.

Referenced by ForeignStorageBuffer::append(), Buffer_Namespace::Buffer::append(), File_Namespace::FileBuffer::append(), Buffer_Namespace::Buffer::write(), and File_Namespace::FileBuffer::write().

                            {
    is_appended_ = true;
    is_dirty_ = true;
  }

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::setDirty ( )

inline

Definition at line 108 of file AbstractBuffer.h.

References is_dirty_.

Referenced by StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), File_Namespace::FileMgr::init(), Buffer_Namespace::Buffer::write(), and File_Namespace::FileBuffer::write().

{ is_dirty_ = true; }

Here is the caller graph for this function:

virtual void Data_Namespace::AbstractBuffer::setMemoryPtr ( int8_t *  new_ptr )

inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 83 of file AbstractBuffer.h.

References CHECK.

Referenced by ForeignStorageBufferMgr::fetchBuffer().

{ CHECK(false); }

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::setSize ( const size_t  size )

inline

Definition at line 120 of file AbstractBuffer.h.

References size(), and size_.

Referenced by copyTo(), ForeignStorageBufferMgr::fetchBuffer(), File_Namespace::FileMgr::init(), File_Namespace::FileMgr::putBuffer(), and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer().

{ size_ = size; }

Here is the call graph for this function:

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::setSqlType ( const SQLTypeInfo &  sql_type )

inline

Definition at line 102 of file AbstractBuffer.h.

References sql_type_.

{ sql_type_ = sql_type; }

void Data_Namespace::AbstractBuffer::setUpdated ( )

inline

Definition at line 110 of file AbstractBuffer.h.

References is_dirty_, and is_updated_.

Referenced by foreign_storage::ForeignStorageCache::cacheChunk(), foreign_storage::ForeignStorageCache::cacheMetadataVec(), foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), Buffer_Namespace::Buffer::write(), and File_Namespace::FileBuffer::write().

                           {
    is_updated_ = true;
    is_dirty_ = true;
  }

Here is the caller graph for this function:

size_t Data_Namespace::AbstractBuffer::size ( ) const

inline

Definition at line 95 of file AbstractBuffer.h.

References size_.

Referenced by foreign_storage::ParquetStringNoneEncoder::appendData(), StringNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset(), Chunk_NS::Chunk::begin_iterator(), Data_Namespace::DataMgr::copy(), copyTo(), Buffer_Namespace::BufferMgr::fetchBuffer(), File_Namespace::FileMgr::fetchBuffer(), Encoder::getMetadata(), File_Namespace::FileMgr::init(), foreign_storage::CsvDataWrapper::populateChunkMapForColumns(), Buffer_Namespace::BufferMgr::putBuffer(), File_Namespace::FileMgr::putBuffer(), foreign_storage::ParquetVariableLengthArrayEncoder::setFirstOffsetForBuffer(), setSize(), and foreign_storage::ParquetStringNoneEncoder::writeInitialOffsetIfApplicable().

{ return size_; }

Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::syncEncoder

(

const AbstractBuffer *

src_buffer

)

Definition at line 33 of file AbstractBuffer.cpp.

References encoder_, hasEncoder(), initEncoder(), and sql_type_.

Referenced by copyTo(), ForeignStorageBufferMgr::fetchBuffer(), File_Namespace::FileMgr::init(), Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

                                                                 {
  if (src_buffer->hasEncoder()) {
    if (!hasEncoder()) {
      initEncoder(src_buffer->sql_type_);
    }
    encoder_->copyMetadata(src_buffer->encoder_.get());
  } else {
    encoder_ = nullptr;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

virtual int Data_Namespace::AbstractBuffer::unPin ( )

inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 91 of file AbstractBuffer.h.

Referenced by Buffer_Namespace::BufferMgr::fetchBuffer(), and Chunk_NS::Chunk::unpinBuffer().

{ return 0; }

Here is the caller graph for this function:

virtual void Data_Namespace::AbstractBuffer::write ( int8_t *  src, const size_t  num_bytes, const size_t  offset = 0, const MemoryLevel  src_buffer_type = CPU_LEVEL, const int  src_device_id = -1  )

pure virtual

Implemented in Buffer_Namespace::Buffer, ForeignStorageBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by DateDaysEncoder< T, V >::appendData(), FixedLengthEncoder< T, V >::appendData(), NoneEncoder< T >::appendData(), Data_Namespace::DataMgr::copy(), and Buffer_Namespace::BufferMgr::putBuffer().

Here is the caller graph for this function:

Member Data Documentation

int Data_Namespace::AbstractBuffer::device_id_

protected

Definition at line 136 of file AbstractBuffer.h.

Referenced by getDeviceId(), Buffer_Namespace::GpuCudaBuffer::readData(), and Buffer_Namespace::GpuCudaBuffer::writeData().

std::unique_ptr<Encoder> Data_Namespace::AbstractBuffer::encoder_

protected

Definition at line 133 of file AbstractBuffer.h.

Referenced by getEncoder(), hasEncoder(), initEncoder(), File_Namespace::FileBuffer::readMetadata(), resetToEmpty(), syncEncoder(), and File_Namespace::FileBuffer::writeMetadata().

bool Data_Namespace::AbstractBuffer::is_appended_

private

Definition at line 140 of file AbstractBuffer.h.

Referenced by clearDirtyBits(), isAppended(), and setAppended().

bool Data_Namespace::AbstractBuffer::is_dirty_

private

Definition at line 139 of file AbstractBuffer.h.

Referenced by clearDirtyBits(), isDirty(), setAppended(), setDirty(), and setUpdated().

bool Data_Namespace::AbstractBuffer::is_updated_

private

Definition at line 141 of file AbstractBuffer.h.

Referenced by clearDirtyBits(), isUpdated(), and setUpdated().

size_t Data_Namespace::AbstractBuffer::size_

protected

Definition at line 135 of file AbstractBuffer.h.

Referenced by foreign_storage::ForeignStorageBuffer::append(), ForeignStorageBuffer::append(), Buffer_Namespace::Buffer::append(), File_Namespace::FileBuffer::append(), Buffer_Namespace::Buffer::read(), File_Namespace::FileBuffer::readMetadata(), foreign_storage::ForeignStorageBuffer::reserve(), resetToEmpty(), setSize(), size(), Buffer_Namespace::Buffer::write(), File_Namespace::FileBuffer::write(), and File_Namespace::FileBuffer::writeMetadata().

SQLTypeInfo Data_Namespace::AbstractBuffer::sql_type_

protected

Definition at line 134 of file AbstractBuffer.h.

Referenced by getSqlType(), initEncoder(), ForeignStorageBuffer::read(), File_Namespace::FileBuffer::readMetadata(), setSqlType(), syncEncoder(), ForeignStorageBuffer::tryZeroCopy(), and File_Namespace::FileBuffer::writeMetadata().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/AbstractBuffer.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/AbstractBuffer.cpp