OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Data_Namespace::AbstractBuffer Class Referenceabstract

An AbstractBuffer is a unit of data management for a data manager. More...

#include <AbstractBuffer.h>

+ Inheritance diagram for Data_Namespace::AbstractBuffer:
+ Collaboration diagram for Data_Namespace::AbstractBuffer:

Public Member Functions

 AbstractBuffer (const int device_id)
 
 AbstractBuffer (const int device_id, const SQLTypeInfo sql_type)
 
virtual ~AbstractBuffer ()
 
virtual void read (int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0
 
virtual void write (int8_t *src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int src_device_id=-1)=0
 
virtual void reserve (size_t num_bytes)=0
 
virtual void append (int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
 
virtual int8_t * getMemoryPtr ()=0
 
virtual void setMemoryPtr (int8_t *new_ptr)
 
virtual size_t pageCount () const =0
 
virtual size_t pageSize () const =0
 
virtual size_t reservedSize () const =0
 
virtual MemoryLevel getType () const =0
 
virtual int pin ()
 
virtual int unPin ()
 
virtual int getPinCount ()
 
size_t size () const
 
int getDeviceId () const
 
bool isDirty () const
 
bool isAppended () const
 
bool isUpdated () const
 
bool hasEncoder () const
 
SQLTypeInfo getSqlType () const
 
void setSqlType (const SQLTypeInfo &sql_type)
 
EncodergetEncoder () const
 
void setDirty ()
 
void setUpdated ()
 
void setAppended ()
 
void setSize (const size_t size)
 
void clearDirtyBits ()
 
void initEncoder (const SQLTypeInfo &tmp_sql_type)
 
void syncEncoder (const AbstractBuffer *src_buffer)
 
void copyTo (AbstractBuffer *destination_buffer, const size_t num_bytes=0)
 
void resetToEmpty ()
 

Protected Attributes

std::unique_ptr< Encoderencoder_
 
SQLTypeInfo sql_type_
 
size_t size_
 
int device_id_
 

Private Attributes

bool is_dirty_
 
bool is_appended_
 
bool is_updated_
 

Detailed Description

An AbstractBuffer is a unit of data management for a data manager.

Definition at line 48 of file AbstractBuffer.h.

Constructor & Destructor Documentation

Data_Namespace::AbstractBuffer::AbstractBuffer ( const int  device_id)
inline

Definition at line 50 of file AbstractBuffer.h.

51  : encoder_(nullptr)
52  , size_(0)
53  , device_id_(device_id)
54  , is_dirty_(false)
55  , is_appended_(false)
56  , is_updated_(false) {}
std::unique_ptr< Encoder > encoder_
Data_Namespace::AbstractBuffer::AbstractBuffer ( const int  device_id,
const SQLTypeInfo  sql_type 
)
inline

Definition at line 58 of file AbstractBuffer.h.

References initEncoder().

59  : size_(0)
60  , device_id_(device_id)
61  , is_dirty_(false)
62  , is_appended_(false)
63  , is_updated_(false) {
64  initEncoder(sql_type);
65  }
void initEncoder(const SQLTypeInfo &tmp_sql_type)

+ Here is the call graph for this function:

virtual Data_Namespace::AbstractBuffer::~AbstractBuffer ( )
inlinevirtual

Definition at line 66 of file AbstractBuffer.h.

66 {}

Member Function Documentation

virtual void Data_Namespace::AbstractBuffer::append ( int8_t *  src,
const size_t  num_bytes,
const MemoryLevel  src_buffer_type = CPU_LEVEL,
const int  device_id = -1 
)
pure virtual

Implemented in Buffer_Namespace::Buffer, ForeignStorageBuffer, foreign_storage::PassThroughBuffer, foreign_storage::TypedParquetStorageBuffer< Type >, foreign_storage::TypedParquetStorageBuffer< std::string >, foreign_storage::TypedParquetStorageBuffer< ArrayDatum >, foreign_storage::TypedParquetDetectBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by foreign_storage::ParquetVariableLengthArrayEncoder::appendArraysToBuffer(), foreign_storage::ParquetArrayEncoder::appendArraysToBuffer(), foreign_storage::ParquetStringNoneEncoder::appendData(), foreign_storage::ParquetInPlaceEncoder::appendData(), StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::appendData(), DateDaysEncoder< T, V >::appendEncodedOrUnencodedData(), FixedLengthEncoder< T, V >::appendEncodedOrUnencodedData(), NoneEncoder< T >::appendValidatedOrNonValidatedData(), data_conversion::StringViewToStringDictEncoder< IdType >::encodeAndAppendData(), data_conversion::StringViewToScalarEncoder< DataType, MetadataType >::encodeAndAppendData(), ColumnFetcher::linearizeFixedLenArrayColFrags(), ColumnFetcher::linearizeVarLenArrayColFrags(), foreign_storage::ParquetDataWrapper::loadBuffersUsingLazyParquetChunkLoader(), Buffer_Namespace::BufferMgr::putBuffer(), foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::resize_delete_buffer(), and foreign_storage::ParquetStringNoneEncoder::writeInitialOffsetIfApplicable().

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::clearDirtyBits ( )
inline

Definition at line 122 of file AbstractBuffer.h.

References is_appended_, is_dirty_, and is_updated_.

Referenced by Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

122  {
123  is_appended_ = false;
124  is_updated_ = false;
125  is_dirty_ = false;
126  }

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::copyTo ( AbstractBuffer destination_buffer,
const size_t  num_bytes = 0 
)

Definition at line 44 of file AbstractBuffer.cpp.

References CHECK_GE, getDeviceId(), getMemoryPtr(), getType(), isUpdated(), read(), reserve(), setSize(), size(), and syncEncoder().

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), File_Namespace::CachingGlobalFileMgr::fetchBuffer(), Buffer_Namespace::BufferMgr::fetchBuffer(), and File_Namespace::FileMgr::fetchBuffer().

44  {
45  CHECK_GE(size(), num_bytes) << "Attempting to copy more bytes than a buffer contains";
46  size_t chunk_size = (num_bytes == 0) ? size() : num_bytes;
47  destination_buffer->reserve(chunk_size);
48  if (isUpdated()) {
49  read(destination_buffer->getMemoryPtr(),
50  chunk_size,
51  0,
52  destination_buffer->getType(),
53  destination_buffer->getDeviceId());
54  } else {
55  read(destination_buffer->getMemoryPtr() + destination_buffer->size(),
56  chunk_size - destination_buffer->size(),
57  destination_buffer->size(),
58  destination_buffer->getType(),
59  destination_buffer->getDeviceId());
60  }
61  destination_buffer->setSize(chunk_size);
62  destination_buffer->syncEncoder(this);
63 }
#define CHECK_GE(x, y)
Definition: Logger.h:306
virtual void read(int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int Data_Namespace::AbstractBuffer::getDeviceId ( ) const
inline

Definition at line 97 of file AbstractBuffer.h.

References device_id_.

Referenced by Data_Namespace::DataMgr::copy(), copyTo(), Data_Namespace::DataMgr::free(), Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

+ Here is the caller graph for this function:

Encoder* Data_Namespace::AbstractBuffer::getEncoder ( ) const
inline

Definition at line 104 of file AbstractBuffer.h.

References CHECK, encoder_, and hasEncoder().

Referenced by data_conversion::StringViewToArrayEncoder< ScalarEncoderType >::appendArrayDatums(), Chunk_NS::Chunk::appendData(), Chunk_NS::Chunk::appendEncodedData(), Chunk_NS::Chunk::appendEncodedDataAtIndices(), foreign_storage::ForeignStorageCache::cacheMetadataVec(), Fragmenter_Namespace::compute_row_indices_of_shards(), data_conversion::StringViewToStringNoneEncoder::encodeAndAppendData(), foreign_storage::get_placeholder_metadata(), foreign_storage::Csv::get_placeholder_metadata(), data_conversion::StringViewToStringNoneEncoder::getEncoder(), Chunk_NS::Chunk::getNumElemsForBytesEncodedDataAtIndices(), Chunk_NS::Chunk::getNumElemsForBytesInsertData(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::getUpdatedStats(), Chunk_NS::Chunk::initEncoder(), DBHandler::insert_chunks(), foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::reduce_metadata(), foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and Chunk_NS::Chunk::setChunkBuffer().

104  {
105  CHECK(hasEncoder());
106  return encoder_.get();
107  }
std::unique_ptr< Encoder > encoder_
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

virtual int8_t* Data_Namespace::AbstractBuffer::getMemoryPtr ( )
pure virtual

Implemented in File_Namespace::FileBuffer, Buffer_Namespace::Buffer, ForeignStorageBuffer, foreign_storage::TypedParquetStorageBuffer< Type >, foreign_storage::TypedParquetStorageBuffer< std::string >, foreign_storage::TypedParquetStorageBuffer< ArrayDatum >, foreign_storage::PassThroughBuffer, foreign_storage::TypedParquetDetectBuffer, and foreign_storage::ForeignStorageBuffer.

Referenced by ThrustAllocator::allocate(), ThrustAllocator::allocateScopedBuffer(), Chunk_NS::Chunk::appendEncodedData(), Chunk_NS::Chunk::appendEncodedDataAtIndices(), Fragmenter_Namespace::ArrayChunkConverter::ArrayChunkConverter(), Chunk_NS::Chunk::begin_iterator(), Data_Namespace::DataMgr::copy(), copyTo(), Fragmenter_Namespace::DateChunkConverter< BUFFER_DATA_TYPE >::DateChunkConverter(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::eraseInvalidIndicesInBuffer(), ForeignStorageBufferMgr::fetchBuffer(), Fragmenter_Namespace::FixedLenArrayChunkConverter::FixedLenArrayChunkConverter(), PerfectHashTable::getGpuBuffer(), BaselineHashTable::getGpuBuffer(), ColumnFetcher::linearizeVarLenArrayColFrags(), foreign_storage::ParquetDataWrapper::loadBuffersUsingLazyParquetChunkLoader(), foreign_storage::AbstractTextFileDataWrapper::populateChunks(), ColumnFetcher::prepareChunkIter(), Buffer_Namespace::BufferMgr::putBuffer(), File_Namespace::FileMgr::putBuffer(), Fragmenter_Namespace::ScalarChunkConverter< BUFFER_DATA_TYPE, INSERT_DATA_TYPE >::ScalarChunkConverter(), and Fragmenter_Namespace::StringChunkConverter::StringChunkConverter().

+ Here is the caller graph for this function:

virtual int Data_Namespace::AbstractBuffer::getPinCount ( )
inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 93 of file AbstractBuffer.h.

Referenced by ThrustAllocator::allocate(), and ThrustAllocator::allocateScopedBuffer().

93 { return 0; }

+ Here is the caller graph for this function:

SQLTypeInfo Data_Namespace::AbstractBuffer::getSqlType ( ) const
inline

Definition at line 102 of file AbstractBuffer.h.

References sql_type_.

Referenced by Encoder::getMetadata(), FixedLengthArrayNoneEncoder::is_null(), FixedLengthArrayNoneEncoder::is_null_ignore_not_null(), FixedLengthArrayNoneEncoder::resetChunkStats(), ArrayNoneEncoder::resetChunkStats(), FixedLengthArrayNoneEncoder::update_elem_stats(), and ArrayNoneEncoder::update_elem_stats().

102 { return sql_type_; }

+ Here is the caller graph for this function:

bool Data_Namespace::AbstractBuffer::hasEncoder ( ) const
inline

Definition at line 101 of file AbstractBuffer.h.

References encoder_.

Referenced by File_Namespace::FileBuffer::dump(), getEncoder(), syncEncoder(), and File_Namespace::FileBuffer::writeMetadata().

101 { return (encoder_ != nullptr); }
std::unique_ptr< Encoder > encoder_

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::initEncoder ( const SQLTypeInfo tmp_sql_type)

Definition at line 26 of file AbstractBuffer.cpp.

References Encoder::Create(), encoder_, logger::FATAL, SQLTypeInfo::get_type_name(), LOG_IF, and sql_type_.

Referenced by AbstractBuffer(), foreign_storage::ParquetEncoder::createMetadata(), foreign_storage::get_placeholder_metadata(), foreign_storage::Csv::get_placeholder_metadata(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::getUpdatedStats(), Chunk_NS::Chunk::initEncoder(), DBHandler::insert_chunks(), File_Namespace::FileBuffer::readMetadata(), foreign_storage::anonymous_namespace{ParquetDataWrapper.cpp}::reduce_metadata(), foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), and syncEncoder().

26  {
27  sql_type_ = tmp_sql_type;
28  encoder_.reset(Encoder::Create(this, sql_type_));
29  LOG_IF(FATAL, encoder_ == nullptr)
30  << "Failed to create encoder for SQL Type " << sql_type_.get_type_name();
31 }
static Encoder * Create(Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
Definition: Encoder.cpp:26
#define LOG_IF(severity, condition)
Definition: Logger.h:384
std::string get_type_name() const
Definition: sqltypes.h:482
std::unique_ptr< Encoder > encoder_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Data_Namespace::AbstractBuffer::isAppended ( ) const
inline

Definition at line 99 of file AbstractBuffer.h.

References is_appended_.

Referenced by Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

+ Here is the caller graph for this function:

bool Data_Namespace::AbstractBuffer::isUpdated ( ) const
inline

Definition at line 100 of file AbstractBuffer.h.

References is_updated_.

Referenced by copyTo(), Buffer_Namespace::BufferMgr::putBuffer(), and File_Namespace::FileMgr::putBuffer().

+ Here is the caller graph for this function:

virtual int Data_Namespace::AbstractBuffer::pin ( )
inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 91 of file AbstractBuffer.h.

Referenced by Buffer_Namespace::BufferMgr::fetchBuffer().

91 { return 0; }

+ Here is the caller graph for this function:

virtual void Data_Namespace::AbstractBuffer::read ( int8_t *const  dst,
const size_t  num_bytes,
const size_t  offset = 0,
const MemoryLevel  dst_buffer_type = CPU_LEVEL,
const int  dst_device_id = -1 
)
pure virtual
virtual size_t Data_Namespace::AbstractBuffer::reservedSize ( ) const
pure virtual
void Data_Namespace::AbstractBuffer::resetToEmpty ( )

Definition at line 65 of file AbstractBuffer.cpp.

References encoder_, and size_.

Referenced by data_conversion::StringViewToScalarEncoder< DataType, MetadataType >::clear(), and data_conversion::StringViewToStringDictEncoder< IdType >::clear().

65  {
66  encoder_ = nullptr;
67  size_ = 0;
68 }
std::unique_ptr< Encoder > encoder_

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::setAppended ( )
inline

Definition at line 116 of file AbstractBuffer.h.

References is_appended_, and is_dirty_.

Referenced by ForeignStorageBuffer::append(), Buffer_Namespace::Buffer::append(), File_Namespace::FileBuffer::append(), File_Namespace::CachingFileMgr::putBuffer(), Buffer_Namespace::Buffer::write(), and File_Namespace::FileBuffer::write().

116  {
117  is_appended_ = true;
118  is_dirty_ = true;
119  }

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::setDirty ( )
inline

Definition at line 109 of file AbstractBuffer.h.

References is_dirty_.

Referenced by StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), File_Namespace::CachingFileMgr::putBuffer(), Buffer_Namespace::Buffer::write(), and File_Namespace::FileBuffer::write().

+ Here is the caller graph for this function:

virtual void Data_Namespace::AbstractBuffer::setMemoryPtr ( int8_t *  new_ptr)
inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 84 of file AbstractBuffer.h.

References CHECK.

Referenced by ForeignStorageBufferMgr::fetchBuffer().

84 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::setSize ( const size_t  size)
inline

Definition at line 121 of file AbstractBuffer.h.

References size(), and size_.

Referenced by copyTo(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::eraseInvalidIndicesInBuffer(), ForeignStorageBufferMgr::fetchBuffer(), foreign_storage::PassThroughBuffer::PassThroughBuffer(), and foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer().

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::setSqlType ( const SQLTypeInfo sql_type)
inline

Definition at line 103 of file AbstractBuffer.h.

References sql_type_.

103 { sql_type_ = sql_type; }
void Data_Namespace::AbstractBuffer::setUpdated ( )
inline

Definition at line 111 of file AbstractBuffer.h.

References is_dirty_, and is_updated_.

Referenced by foreign_storage::ForeignStorageCache::cacheMetadataVec(), foreign_storage::anonymous_namespace{ForeignStorageCache.cpp}::set_metadata_for_buffer(), Buffer_Namespace::Buffer::write(), and File_Namespace::FileBuffer::write().

111  {
112  is_updated_ = true;
113  is_dirty_ = true;
114  }

+ Here is the caller graph for this function:

size_t Data_Namespace::AbstractBuffer::size ( ) const
inline

Definition at line 96 of file AbstractBuffer.h.

References size_.

Referenced by foreign_storage::ParquetStringNoneEncoder::appendData(), StringNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), DateDaysEncoder< T, V >::appendEncodedOrUnencodedData(), FixedLengthEncoder< T, V >::appendEncodedOrUnencodedData(), foreign_storage::ParquetVariableLengthArrayEncoder::appendLastArrayOffset(), NoneEncoder< T >::appendValidatedOrNonValidatedData(), Chunk_NS::Chunk::begin_iterator(), Data_Namespace::DataMgr::copy(), copyTo(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::eraseInvalidIndicesInBuffer(), Buffer_Namespace::BufferMgr::fetchBuffer(), File_Namespace::FileMgr::fetchBuffer(), data_conversion::BaseConvertEncoder::getMetadata(), Encoder::getMetadata(), foreign_storage::init_chunk_for_column(), foreign_storage::Csv::init_chunk_for_column(), File_Namespace::FileBuffer::isMissingPages(), foreign_storage::ParquetDataWrapper::loadBuffersUsingLazyParquetChunkLoader(), ColumnFetcher::prepareChunkIter(), Buffer_Namespace::BufferMgr::putBuffer(), File_Namespace::FileMgr::putBuffer(), File_Namespace::CachingFileMgr::putBuffer(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::reserve(), foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::resize_delete_buffer(), foreign_storage::ParquetVariableLengthArrayEncoder::setFirstOffsetForBuffer(), setSize(), and foreign_storage::ParquetStringNoneEncoder::writeInitialOffsetIfApplicable().

96 { return size_; }

+ Here is the caller graph for this function:

void Data_Namespace::AbstractBuffer::syncEncoder ( const AbstractBuffer src_buffer)

Definition at line 33 of file AbstractBuffer.cpp.

References encoder_, hasEncoder(), initEncoder(), and sql_type_.

Referenced by copyTo(), ForeignStorageBufferMgr::fetchBuffer(), File_Namespace::FileMgr::init(), and Buffer_Namespace::BufferMgr::putBuffer().

33  {
34  if (src_buffer->hasEncoder()) {
35  if (!hasEncoder()) {
36  initEncoder(src_buffer->sql_type_);
37  }
38  encoder_->copyMetadata(src_buffer->encoder_.get());
39  } else {
40  encoder_ = nullptr;
41  }
42 }
void initEncoder(const SQLTypeInfo &tmp_sql_type)
std::unique_ptr< Encoder > encoder_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

virtual int Data_Namespace::AbstractBuffer::unPin ( )
inlinevirtual

Reimplemented in Buffer_Namespace::Buffer.

Definition at line 92 of file AbstractBuffer.h.

Referenced by Buffer_Namespace::BufferMgr::fetchBuffer(), Chunk_NS::Chunk::unpinBuffer(), and Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::BaseAlterColumnContext::unpinChunk().

92 { return 0; }

+ Here is the caller graph for this function:

virtual void Data_Namespace::AbstractBuffer::write ( int8_t *  src,
const size_t  num_bytes,
const size_t  offset = 0,
const MemoryLevel  src_buffer_type = CPU_LEVEL,
const int  src_device_id = -1 
)
pure virtual

Member Data Documentation

int Data_Namespace::AbstractBuffer::device_id_
protected
std::unique_ptr<Encoder> Data_Namespace::AbstractBuffer::encoder_
protected
bool Data_Namespace::AbstractBuffer::is_appended_
private

Definition at line 141 of file AbstractBuffer.h.

Referenced by clearDirtyBits(), isAppended(), and setAppended().

bool Data_Namespace::AbstractBuffer::is_dirty_
private

Definition at line 140 of file AbstractBuffer.h.

Referenced by clearDirtyBits(), isDirty(), setAppended(), setDirty(), and setUpdated().

bool Data_Namespace::AbstractBuffer::is_updated_
private

Definition at line 142 of file AbstractBuffer.h.

Referenced by clearDirtyBits(), isUpdated(), and setUpdated().


The documentation for this class was generated from the following files: