Chunk_NS::Chunk Class Reference

#include <Chunk.h>

Collaboration diagram for Chunk_NS::Chunk:

Public Member Functions
	Chunk ()
	Chunk (const ColumnDescriptor *td)
	Chunk (AbstractBuffer *b, AbstractBuffer *ib, const ColumnDescriptor *td)
	~Chunk ()
const ColumnDescriptor *	getColumnDesc () const
ChunkIter	begin_iterator (const std::shared_ptr< ChunkMetadata > &, int start_idx=0, int skip=1) const
size_t	getNumElemsForBytesInsertData (const DataBlockPtr &src_data, const size_t num_elems, const size_t start_idx, const size_t byte_limit, const bool replicating=false)
std::shared_ptr< ChunkMetadata >	appendData (DataBlockPtr &srcData, const size_t numAppendElems, const size_t startIdx, const bool replicating=false)
void	createChunkBuffer (DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t page_size=0)
void	getChunkBuffer (DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t num_bytes=0, const size_t num_elems=0)
bool	isChunkOnDevice (DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int device_id)
AbstractBuffer *	getBuffer () const
AbstractBuffer *	getIndexBuf () const
void	setBuffer (AbstractBuffer *b)
void	setIndexBuffer (AbstractBuffer *ib)
void	initEncoder ()
void	decompress (int8_t *compressed, VarlenDatum *result, Datum *datum) const

Static Public Member Functions
static void	translateColumnDescriptorsToChunkVec (const std::list< const ColumnDescriptor *> &colDescs, std::vector< Chunk > &chunkVec)
static std::shared_ptr< Chunk >	getChunk (const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)

Private Member Functions
void	unpinBuffer ()

Private Attributes
AbstractBuffer *	buffer_
AbstractBuffer *	index_buf_
const ColumnDescriptor *	column_desc_

Detailed Description

Definition at line 41 of file Chunk.h.

Constructor & Destructor Documentation

Chunk() [1/3]

Chunk_NS::Chunk::Chunk ( )

inline

Definition at line 43 of file Chunk.h.

Referenced by getChunk().

: buffer_(nullptr), index_buf_(nullptr), column_desc_(nullptr) {}

Here is the caller graph for this function:

Chunk() [2/3]

Chunk_NS::Chunk::Chunk ( const ColumnDescriptor *  td )

inlineexplicit

Definition at line 45 of file Chunk.h.

      : buffer_(nullptr), index_buf_(nullptr), column_desc_(td) {}

Chunk() [3/3]

Chunk_NS::Chunk::Chunk ( AbstractBuffer *  b, AbstractBuffer *  ib, const ColumnDescriptor *  td  )

inline

Definition at line 48 of file Chunk.h.

      : buffer_(b), index_buf_(ib), column_desc_(td){};

~Chunk()

Chunk_NS::Chunk::~Chunk ( )

inline

Definition at line 51 of file Chunk.h.

References unpinBuffer().

{ unpinBuffer(); }

Here is the call graph for this function:

Member Function Documentation

appendData()

std::shared_ptr< ChunkMetadata > Chunk_NS::Chunk::appendData	(	DataBlockPtr &	srcData,
		const size_t	numAppendElems,
		const size_t	startIdx,
		const bool	replicating = false
	)

Definition at line 171 of file Chunk.cpp.

References StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), Encoder::appendData(), DataBlockPtr::arraysPtr, buffer_, CHECK, CHECK_EQ, column_desc_, ColumnDescriptor::columnType, Data_Namespace::AbstractBuffer::getEncoder(), kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, DataBlockPtr::numbersPtr, and DataBlockPtr::stringsPtr.

Referenced by translateColumnDescriptorsToChunkVec().

                                                                         {
  const auto& ti = column_desc_->columnType;
  if (ti.is_varlen()) {
    switch (ti.get_type()) {
      case kARRAY: {
        if (ti.get_size() > 0) {
          FixedLengthArrayNoneEncoder* array_encoder =
              dynamic_cast<FixedLengthArrayNoneEncoder*>(buffer_->getEncoder());
          return array_encoder->appendData(
              src_data.arraysPtr, start_idx, num_elems, replicating);
        }
        ArrayNoneEncoder* array_encoder =
            dynamic_cast<ArrayNoneEncoder*>(buffer_->getEncoder());
        return array_encoder->appendData(
            src_data.arraysPtr, start_idx, num_elems, replicating);
      }
      case kTEXT:
      case kVARCHAR:
      case kCHAR: {
        CHECK_EQ(kENCODING_NONE, ti.get_compression());
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
        return str_encoder->appendData(
            src_data.stringsPtr, start_idx, num_elems, replicating);
      }
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON: {
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
        return str_encoder->appendData(
            src_data.stringsPtr, start_idx, num_elems, replicating);
      }
      default:
        CHECK(false);
    }
  }
  return buffer_->getEncoder()->appendData(
      src_data.numbersPtr, num_elems, ti, replicating);
}

Here is the call graph for this function:

Here is the caller graph for this function:

begin_iterator()

ChunkIter Chunk_NS::Chunk::begin_iterator	(	const std::shared_ptr< ChunkMetadata > &	chunk_metadata,
		int	start_idx = 0,
		int	skip = 1
	)		const

Definition at line 260 of file Chunk.cpp.

References buffer_, column_desc_, ColumnDescriptor::columnType, ChunkIter::current_pos, ChunkIter::end_pos, SQLTypeInfo::get_size(), Data_Namespace::AbstractBuffer::getMemoryPtr(), index_buf_, ChunkIter::num_elems, ChunkIter::second_buf, Data_Namespace::AbstractBuffer::size(), ChunkIter::skip, ChunkIter::skip_size, ChunkIter::start_pos, and ChunkIter::type_info.

Referenced by translateColumnDescriptorsToChunkVec().

                                                {
  ChunkIter it;
  it.type_info = column_desc_->columnType;
  it.skip = skip;
  it.skip_size = column_desc_->columnType.get_size();
  if (it.skip_size < 0) {  // if it's variable length
    it.current_pos = it.start_pos =
        index_buf_->getMemoryPtr() + start_idx * sizeof(StringOffsetT);
    it.end_pos = index_buf_->getMemoryPtr() + index_buf_->size() - sizeof(StringOffsetT);
    it.second_buf = buffer_->getMemoryPtr();
  } else {
    it.current_pos = it.start_pos = buffer_->getMemoryPtr() + start_idx * it.skip_size;
    it.end_pos = buffer_->getMemoryPtr() + buffer_->size();
    it.second_buf = nullptr;
  }
  it.num_elems = chunk_metadata->numElements;
  return it;
}

Here is the call graph for this function:

Here is the caller graph for this function:

createChunkBuffer()

void Chunk_NS::Chunk::createChunkBuffer	(	DataMgr *	data_mgr,
		const ChunkKey &	key,
		const MemoryLevel	mem_level,
		const int	deviceId = 0,
		const size_t	page_size = 0
	)

Definition at line 110 of file Chunk.cpp.

References buffer_, column_desc_, ColumnDescriptor::columnType, Data_Namespace::DataMgr::createChunkBuffer(), index_buf_, SQLTypeInfo::is_fixlen_array(), and SQLTypeInfo::is_varlen().

Referenced by translateColumnDescriptorsToChunkVec().

                                                      {
  if (column_desc_->columnType.is_varlen() &&
      !column_desc_->columnType.is_fixlen_array()) {
    ChunkKey subKey = key;
    subKey.push_back(1);  // 1 for the main buffer_
    buffer_ = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size);
    subKey.pop_back();
    subKey.push_back(2);  // 2 for the index buffer_
    index_buf_ = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size);
  } else {
    buffer_ = data_mgr->createChunkBuffer(key, mem_level, device_id, page_size);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

decompress()

void Chunk_NS::Chunk::decompress	(	int8_t *	compressed,
		VarlenDatum *	result,
		Datum *	datum
	)		const

Referenced by setIndexBuffer().

Here is the caller graph for this function:

getBuffer()

AbstractBuffer* Chunk_NS::Chunk::getBuffer ( ) const

inline

Definition at line 104 of file Chunk.h.

References buffer_.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_string_encoder(), Fragmenter_Namespace::DateChunkConverter< BUFFER_DATA_TYPE >::DateChunkConverter(), Fragmenter_Namespace::FixedLenArrayChunkConverter::FixedLenArrayChunkConverter(), Fragmenter_Namespace::ScalarChunkConverter< BUFFER_DATA_TYPE, INSERT_DATA_TYPE >::ScalarChunkConverter(), and Fragmenter_Namespace::StringChunkConverter::StringChunkConverter().

{ return buffer_; }

Here is the caller graph for this function:

getChunk()

std::shared_ptr< Chunk > Chunk_NS::Chunk::getChunk ( const ColumnDescriptor *  cd, DataMgr *  data_mgr, const ChunkKey &  key, const MemoryLevel  mem_level, const int  deviceId, const size_t  num_bytes, const size_t  num_elems  )

static

Definition at line 28 of file Chunk.cpp.

References Chunk().

Referenced by Fragmenter_Namespace::get_chunks(), getChunkAccessorTable(), Fragmenter_Namespace::InsertOrderFragmenter::getChunksForAllColumns(), ColumnFetcher::getOneColumnFragment(), ColumnFetcher::getOneTableColumnFragment(), translateColumnDescriptorsToChunkVec(), Fragmenter_Namespace::InsertOrderFragmenter::updateChunkStats(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Catalog_Namespace::Catalog::vacuumDeletedRows(), StorageIOFacility< RelAlgExecutorTraits >::yieldDeleteCallback(), and StorageIOFacility< RelAlgExecutorTraits >::yieldUpdateCallback().

                                                              {
  std::shared_ptr<Chunk> chunkp = std::make_shared<Chunk>(Chunk(cd));
  chunkp->getChunkBuffer(data_mgr, key, memoryLevel, deviceId, numBytes, numElems);
  return chunkp;
}

Here is the call graph for this function:

Here is the caller graph for this function:

getChunkBuffer()

void Chunk_NS::Chunk::getChunkBuffer	(	DataMgr *	data_mgr,
		const ChunkKey &	key,
		const MemoryLevel	mem_level,
		const int	deviceId = 0,
		const size_t	num_bytes = 0,
		const size_t	num_elems = 0
	)

Definition at line 58 of file Chunk.cpp.

References buffer_, CHECK, CHECK_EQ, column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_type(), Data_Namespace::DataMgr::getChunkBuffer(), Data_Namespace::AbstractBuffer::getEncoder(), index_buf_, SQLTypeInfo::is_fixlen_array(), SQLTypeInfo::is_varlen(), kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, and UNREACHABLE.

Referenced by translateColumnDescriptorsToChunkVec().

                                                   {
  if (column_desc_->columnType.is_varlen() &&
      !column_desc_->columnType.is_fixlen_array()) {
    ChunkKey subKey = key;
    subKey.push_back(1);  // 1 for the main buffer_
    buffer_ = data_mgr->getChunkBuffer(subKey, mem_level, device_id, num_bytes);
    subKey.pop_back();
    subKey.push_back(2);  // 2 for the index buffer_
    index_buf_ = data_mgr->getChunkBuffer(
        subKey,
        mem_level,
        device_id,
        (num_elems + 1) * sizeof(StringOffsetT));  // always record n+1 offsets so string
                                                   // length can be calculated
    switch (column_desc_->columnType.get_type()) {
      case kARRAY: {
        auto array_encoder = dynamic_cast<ArrayNoneEncoder*>(buffer_->getEncoder());
        CHECK(array_encoder);
        array_encoder->setIndexBuffer(index_buf_);
        break;
      }
      case kTEXT:
      case kVARCHAR:
      case kCHAR: {
        CHECK_EQ(kENCODING_NONE, column_desc_->columnType.get_compression());
        auto str_encoder = dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
        CHECK(str_encoder);
        str_encoder->setIndexBuffer(index_buf_);
        break;
      }
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON: {
        auto str_encoder = dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
        CHECK(str_encoder);
        str_encoder->setIndexBuffer(index_buf_);
        break;
      }
      default:
        UNREACHABLE();
    }
  } else {
    buffer_ = data_mgr->getChunkBuffer(key, mem_level, device_id, num_bytes);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

getColumnDesc()

const ColumnDescriptor* Chunk_NS::Chunk::getColumnDesc ( ) const

inline

Definition at line 53 of file Chunk.h.

References column_desc_.

Referenced by Fragmenter_Namespace::FixedLenArrayChunkConverter::FixedLenArrayChunkConverter(), and anonymous_namespace{ExecutionKernel.cpp}::need_to_hold_chunk().

{ return column_desc_; }

Here is the caller graph for this function:

getIndexBuf()

AbstractBuffer* Chunk_NS::Chunk::getIndexBuf ( ) const

inline

Definition at line 106 of file Chunk.h.

References index_buf_.

Referenced by Fragmenter_Namespace::ArrayChunkConverter::ArrayChunkConverter(), foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_string_encoder(), and Fragmenter_Namespace::StringChunkConverter::StringChunkConverter().

{ return index_buf_; }

Here is the caller graph for this function:

getNumElemsForBytesInsertData()

size_t Chunk_NS::Chunk::getNumElemsForBytesInsertData	(	const DataBlockPtr &	src_data,
		const size_t	num_elems,
		const size_t	start_idx,
		const size_t	byte_limit,
		const bool	replicating = false
	)

Definition at line 128 of file Chunk.cpp.

References DataBlockPtr::arraysPtr, buffer_, CHECK, CHECK_EQ, column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), Data_Namespace::AbstractBuffer::getEncoder(), StringNoneEncoder::getNumElemsForBytesInsertData(), FixedLengthArrayNoneEncoder::getNumElemsForBytesInsertData(), ArrayNoneEncoder::getNumElemsForBytesInsertData(), SQLTypeInfo::is_varlen(), kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, and DataBlockPtr::stringsPtr.

Referenced by translateColumnDescriptorsToChunkVec().

                                                                    {
  CHECK(column_desc_->columnType.is_varlen());
  switch (column_desc_->columnType.get_type()) {
    case kARRAY: {
      if (column_desc_->columnType.get_size() > 0) {
        FixedLengthArrayNoneEncoder* array_encoder =
            dynamic_cast<FixedLengthArrayNoneEncoder*>(buffer_->getEncoder());
        return array_encoder->getNumElemsForBytesInsertData(
            src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating);
      }
      ArrayNoneEncoder* array_encoder =
          dynamic_cast<ArrayNoneEncoder*>(buffer_->getEncoder());
      return array_encoder->getNumElemsForBytesInsertData(
          src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating);
    }
    case kTEXT:
    case kVARCHAR:
    case kCHAR: {
      CHECK_EQ(kENCODING_NONE, column_desc_->columnType.get_compression());
      StringNoneEncoder* str_encoder =
          dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
      return str_encoder->getNumElemsForBytesInsertData(
          src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating);
    }
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON: {
      StringNoneEncoder* str_encoder =
          dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
      return str_encoder->getNumElemsForBytesInsertData(
          src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating);
    }
    default:
      CHECK(false);
      return 0;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

initEncoder()

void Chunk_NS::Chunk::initEncoder

(

)

Definition at line 225 of file Chunk.cpp.

References buffer_, CHECK, CHECK_EQ, column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_type(), Data_Namespace::AbstractBuffer::getEncoder(), index_buf_, Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfo::is_fixlen_array(), SQLTypeInfo::is_varlen(), kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, StringNoneEncoder::setIndexBuffer(), and ArrayNoneEncoder::setIndexBuffer().

Referenced by setIndexBuffer().

                        {
  buffer_->initEncoder(column_desc_->columnType);
  if (column_desc_->columnType.is_varlen() &&
      !column_desc_->columnType.is_fixlen_array()) {
    switch (column_desc_->columnType.get_type()) {
      case kARRAY: {
        ArrayNoneEncoder* array_encoder =
            dynamic_cast<ArrayNoneEncoder*>(buffer_->getEncoder());
        array_encoder->setIndexBuffer(index_buf_);
        break;
      }
      case kTEXT:
      case kVARCHAR:
      case kCHAR: {
        CHECK_EQ(kENCODING_NONE, column_desc_->columnType.get_compression());
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
        str_encoder->setIndexBuffer(index_buf_);
        break;
      }
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON: {
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer_->getEncoder());
        str_encoder->setIndexBuffer(index_buf_);
        break;
      }
      default:
        CHECK(false);
    }
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

isChunkOnDevice()

bool Chunk_NS::Chunk::isChunkOnDevice	(	DataMgr *	data_mgr,
		const ChunkKey &	key,
		const MemoryLevel	mem_level,
		const int	device_id
	)

Definition at line 40 of file Chunk.cpp.

References column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::is_fixlen_array(), SQLTypeInfo::is_varlen(), and Data_Namespace::DataMgr::isBufferOnDevice().

Referenced by translateColumnDescriptorsToChunkVec().

                                                 {
  if (column_desc_->columnType.is_varlen() &&
      !column_desc_->columnType.is_fixlen_array()) {
    ChunkKey subKey = key;
    ChunkKey indexKey(subKey);
    indexKey.push_back(1);
    ChunkKey dataKey(subKey);
    dataKey.push_back(2);
    return data_mgr->isBufferOnDevice(indexKey, mem_level, device_id) &&
           data_mgr->isBufferOnDevice(dataKey, mem_level, device_id);
  } else {
    return data_mgr->isBufferOnDevice(key, mem_level, device_id);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

setBuffer()

void Chunk_NS::Chunk::setBuffer ( AbstractBuffer *  b )

inline

Definition at line 108 of file Chunk.h.

References buffer_.

Referenced by foreign_storage::CsvDataWrapper::populateChunkMapForColumns().

{ buffer_ = b; }

Here is the caller graph for this function:

setIndexBuffer()

void Chunk_NS::Chunk::setIndexBuffer ( AbstractBuffer *  ib )

inline

Definition at line 110 of file Chunk.h.

References decompress(), index_buf_, initEncoder(), and run_benchmark_import::result.

{ index_buf_ = ib; }

Here is the call graph for this function:

translateColumnDescriptorsToChunkVec()

static void Chunk_NS::Chunk::translateColumnDescriptorsToChunkVec ( const std::list< const ColumnDescriptor *> &  colDescs, std::vector< Chunk > &  chunkVec  )

inlinestatic

Definition at line 55 of file Chunk.h.

References appendData(), begin_iterator(), createChunkBuffer(), getChunk(), getChunkBuffer(), getNumElemsForBytesInsertData(), and isChunkOnDevice().

                                  {
    for (auto cd : colDescs) {
      chunkVec.emplace_back(cd);
    }
  }

Here is the call graph for this function:

unpinBuffer()

void Chunk_NS::Chunk::unpinBuffer ( )

private

Definition at line 216 of file Chunk.cpp.

References buffer_, index_buf_, and Data_Namespace::AbstractBuffer::unPin().

Referenced by ~Chunk().

                        {
  if (buffer_) {
    buffer_->unPin();
  }
  if (index_buf_) {
    index_buf_->unPin();
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

buffer_

AbstractBuffer* Chunk_NS::Chunk::buffer_

private

Definition at line 117 of file Chunk.h.

Referenced by appendData(), begin_iterator(), createChunkBuffer(), getBuffer(), getChunkBuffer(), getNumElemsForBytesInsertData(), initEncoder(), setBuffer(), and unpinBuffer().

column_desc_

const ColumnDescriptor* Chunk_NS::Chunk::column_desc_

private

Definition at line 119 of file Chunk.h.

Referenced by appendData(), begin_iterator(), createChunkBuffer(), getChunkBuffer(), getColumnDesc(), getNumElemsForBytesInsertData(), initEncoder(), and isChunkOnDevice().

index_buf_

AbstractBuffer* Chunk_NS::Chunk::index_buf_

private

Definition at line 118 of file Chunk.h.

Referenced by begin_iterator(), createChunkBuffer(), getChunkBuffer(), getIndexBuf(), initEncoder(), setIndexBuffer(), and unpinBuffer().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/Chunk/Chunk.h
• /home/jenkins-slave/workspace/core-os-doxygen/DataMgr/Chunk/Chunk.cpp