Chunk_NS::Chunk Class Reference

#include <Chunk.h>

Collaboration diagram for Chunk_NS::Chunk:

Public Member Functions
	Chunk ()
	Chunk (const ColumnDescriptor *td)
	Chunk (AbstractBuffer *b, AbstractBuffer *ib, const ColumnDescriptor *td)
	~Chunk ()
const ColumnDescriptor *	get_column_desc () const
ChunkIter	begin_iterator (const ChunkMetadata &, int start_idx=0, int skip=1) const
size_t	getNumElemsForBytesInsertData (const DataBlockPtr &src_data, const size_t num_elems, const size_t start_idx, const size_t byte_limit, const bool replicating=false)
ChunkMetadata	appendData (DataBlockPtr &srcData, const size_t numAppendElems, const size_t startIdx, const bool replicating=false)
void	createChunkBuffer (DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t page_size=0)
void	getChunkBuffer (DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId=0, const size_t num_bytes=0, const size_t num_elems=0)
bool	isChunkOnDevice (DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int device_id)
AbstractBuffer *	get_buffer () const
AbstractBuffer *	get_index_buf () const
void	set_buffer (AbstractBuffer *b)
void	set_index_buf (AbstractBuffer *ib)
void	init_encoder ()
void	decompress (int8_t *compressed, VarlenDatum *result, Datum *datum) const

Static Public Member Functions
static void	translateColumnDescriptorsToChunkVec (const std::list< const ColumnDescriptor * > &colDescs, std::vector< Chunk > &chunkVec)
static std::shared_ptr< Chunk >	getChunk (const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)

Private Member Functions
void	unpin_buffer ()

Private Attributes
AbstractBuffer *	buffer
AbstractBuffer *	index_buf
const ColumnDescriptor *	column_desc

Detailed Description

Definition at line 44 of file Chunk.h.

Constructor & Destructor Documentation

Chunk_NS::Chunk::Chunk ( )

inline

Definition at line 46 of file Chunk.h.

Referenced by getChunk().

: buffer(nullptr), index_buf(nullptr), column_desc(nullptr) {}

Here is the caller graph for this function:

Chunk_NS::Chunk::Chunk ( const ColumnDescriptor *  td )

inlineexplicit

Definition at line 47 of file Chunk.h.

      : buffer(nullptr), index_buf(nullptr), column_desc(td) {}

Chunk_NS::Chunk::Chunk ( AbstractBuffer *  b, AbstractBuffer *  ib, const ColumnDescriptor *  td  )

inline

Definition at line 49 of file Chunk.h.

      : buffer(b), index_buf(ib), column_desc(td){};

Chunk_NS::Chunk::~Chunk ( )

inline

Definition at line 51 of file Chunk.h.

References unpin_buffer().

{ unpin_buffer(); }

Here is the call graph for this function:

Member Function Documentation

ChunkMetadata Chunk_NS::Chunk::appendData	(	DataBlockPtr &	srcData,
		const size_t	numAppendElems,
		const size_t	startIdx,
		const bool	replicating = false
	)

Definition at line 168 of file Chunk.cpp.

References StringNoneEncoder::appendData(), FixedLengthArrayNoneEncoder::appendData(), ArrayNoneEncoder::appendData(), DataBlockPtr::arraysPtr, buffer, CHECK(), CHECK_EQ, column_desc, ColumnDescriptor::columnType, Data_Namespace::AbstractBuffer::encoder, kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, DataBlockPtr::numbersPtr, and DataBlockPtr::stringsPtr.

                                                        {
  const auto& ti = column_desc->columnType;
  if (ti.is_varlen()) {
    switch (ti.get_type()) {
      case kARRAY: {
        if (ti.get_size() > 0) {
          FixedLengthArrayNoneEncoder* array_encoder =
              dynamic_cast<FixedLengthArrayNoneEncoder*>(buffer->encoder.get());
          return array_encoder->appendData(
              src_data.arraysPtr, start_idx, num_elems, replicating);
        }
        ArrayNoneEncoder* array_encoder =
            dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
        return array_encoder->appendData(
            src_data.arraysPtr, start_idx, num_elems, replicating);
      }
      case kTEXT:
      case kVARCHAR:
      case kCHAR: {
        CHECK_EQ(kENCODING_NONE, ti.get_compression());
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
        return str_encoder->appendData(
            src_data.stringsPtr, start_idx, num_elems, replicating);
      }
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON: {
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
        return str_encoder->appendData(
            src_data.stringsPtr, start_idx, num_elems, replicating);
      }
      default:
        CHECK(false);
    }
  }
  return buffer->encoder->appendData(src_data.numbersPtr, num_elems, ti, replicating);
}

Here is the call graph for this function:

ChunkIter Chunk_NS::Chunk::begin_iterator	(	const ChunkMetadata &	chunk_metadata,
		int	start_idx = 0,
		int	skip = 1
	)		const

Definition at line 255 of file Chunk.cpp.

References buffer, column_desc, ColumnDescriptor::columnType, ChunkIter::current_pos, ChunkIter::end_pos, SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), Data_Namespace::AbstractBuffer::getMemoryPtr(), index_buf, ChunkIter::num_elems, ChunkMetadata::numElements, ChunkIter::second_buf, Data_Namespace::AbstractBuffer::size(), Testing::skip, ChunkIter::skip, ChunkIter::skip_size, ChunkIter::start_pos, and ChunkIter::type_info.

                                                {
  ChunkIter it;
  it.type_info = column_desc->columnType;
  it.skip = skip;
  it.skip_size = column_desc->columnType.get_size();
  if (it.skip_size < 0) {  // if it's variable length
    it.current_pos = it.start_pos =
        index_buf->getMemoryPtr() + start_idx * sizeof(StringOffsetT);
    it.end_pos = index_buf->getMemoryPtr() + index_buf->size() - sizeof(StringOffsetT);
    it.second_buf = buffer->getMemoryPtr();
  } else {
    it.current_pos = it.start_pos = buffer->getMemoryPtr() + start_idx * it.skip_size;
    it.end_pos = buffer->getMemoryPtr() + buffer->size();
    it.second_buf = nullptr;
  }
  it.num_elems = chunk_metadata.numElements;
  return it;
}

Here is the call graph for this function:

void Chunk_NS::Chunk::createChunkBuffer	(	DataMgr *	data_mgr,
		const ChunkKey &	key,
		const MemoryLevel	mem_level,
		const int	deviceId = 0,
		const size_t	page_size = 0
	)

Definition at line 108 of file Chunk.cpp.

References buffer, column_desc, ColumnDescriptor::columnType, Data_Namespace::DataMgr::createChunkBuffer(), index_buf, SQLTypeInfoCore< TYPE_FACET_PACK >::is_fixlen_array(), and SQLTypeInfoCore< TYPE_FACET_PACK >::is_varlen().

                                                      {
  if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) {
    ChunkKey subKey = key;
    subKey.push_back(1);  // 1 for the main buffer
    buffer = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size);
    subKey.pop_back();
    subKey.push_back(2);  // 2 for the index buffer
    index_buf = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size);
  } else {
    buffer = data_mgr->createChunkBuffer(key, mem_level, device_id, page_size);
  }
}

Here is the call graph for this function:

void Chunk_NS::Chunk::decompress	(	int8_t *	compressed,
		VarlenDatum *	result,
		Datum *	datum
	)		const

AbstractBuffer* Chunk_NS::Chunk::get_buffer ( ) const

inline

Definition at line 94 of file Chunk.h.

References buffer.

Referenced by Fragmenter_Namespace::DateChunkConverter< BUFFER_DATA_TYPE >::DateChunkConverter(), Fragmenter_Namespace::FixedLenArrayChunkConverter::FixedLenArrayChunkConverter(), Fragmenter_Namespace::ScalarChunkConverter< BUFFER_DATA_TYPE, INSERT_DATA_TYPE >::ScalarChunkConverter(), and Fragmenter_Namespace::StringChunkConverter::StringChunkConverter().

{ return buffer; }

Here is the caller graph for this function:

const ColumnDescriptor* Chunk_NS::Chunk::get_column_desc ( ) const

inline

Definition at line 52 of file Chunk.h.

References column_desc.

Referenced by Fragmenter_Namespace::FixedLenArrayChunkConverter::FixedLenArrayChunkConverter(), and anonymous_namespace{ExecutionDispatch.cpp}::need_to_hold_chunk().

{ return column_desc; }

Here is the caller graph for this function:

AbstractBuffer* Chunk_NS::Chunk::get_index_buf ( ) const

inline

Definition at line 95 of file Chunk.h.

References index_buf.

Referenced by Fragmenter_Namespace::ArrayChunkConverter::ArrayChunkConverter(), and Fragmenter_Namespace::StringChunkConverter::StringChunkConverter().

{ return index_buf; }

Here is the caller graph for this function:

std::shared_ptr< Chunk > Chunk_NS::Chunk::getChunk ( const ColumnDescriptor *  cd, DataMgr *  data_mgr, const ChunkKey &  key, const MemoryLevel  mem_level, const int  deviceId, const size_t  num_bytes, const size_t  num_elems  )

static

Definition at line 28 of file Chunk.cpp.

References Chunk().

Referenced by Fragmenter_Namespace::get_chunks(), getChunkAccessorTable(), Fragmenter_Namespace::InsertOrderFragmenter::getChunksForAllColumns(), ColumnFetcher::getOneColumnFragment(), ColumnFetcher::getOneTableColumnFragment(), Fragmenter_Namespace::InsertOrderFragmenter::updateChunkStats(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Catalog_Namespace::Catalog::vacuumDeletedRows().

                                                              {
  std::shared_ptr<Chunk> chunkp = std::make_shared<Chunk>(Chunk(cd));
  chunkp->getChunkBuffer(data_mgr, key, memoryLevel, deviceId, numBytes, numElems);
  return chunkp;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void Chunk_NS::Chunk::getChunkBuffer	(	DataMgr *	data_mgr,
		const ChunkKey &	key,
		const MemoryLevel	mem_level,
		const int	deviceId = 0,
		const size_t	num_bytes = 0,
		const size_t	num_elems = 0
	)

Definition at line 57 of file Chunk.cpp.

References buffer, CHECK(), CHECK_EQ, column_desc, ColumnDescriptor::columnType, Data_Namespace::AbstractBuffer::encoder, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), Data_Namespace::DataMgr::getChunkBuffer(), index_buf, SQLTypeInfoCore< TYPE_FACET_PACK >::is_fixlen_array(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_varlen(), kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, StringNoneEncoder::set_index_buf(), and ArrayNoneEncoder::set_index_buf().

                                                   {
  if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) {
    ChunkKey subKey = key;
    subKey.push_back(1);  // 1 for the main buffer
    buffer = data_mgr->getChunkBuffer(subKey, mem_level, device_id, num_bytes);
    subKey.pop_back();
    subKey.push_back(2);  // 2 for the index buffer
    index_buf = data_mgr->getChunkBuffer(
        subKey,
        mem_level,
        device_id,
        (num_elems + 1) * sizeof(StringOffsetT));  // always record n+1 offsets so string
                                                   // length can be calculated
    switch (column_desc->columnType.get_type()) {
      case kARRAY: {
        ArrayNoneEncoder* array_encoder =
            dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
        array_encoder->set_index_buf(index_buf);
        break;
      }
      case kTEXT:
      case kVARCHAR:
      case kCHAR: {
        CHECK_EQ(kENCODING_NONE, column_desc->columnType.get_compression());
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
        str_encoder->set_index_buf(index_buf);
        break;
      }
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON: {
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
        str_encoder->set_index_buf(index_buf);
        break;
      }
      default:
        CHECK(false);
    }
  } else {
    buffer = data_mgr->getChunkBuffer(key, mem_level, device_id, num_bytes);
  }
}

Here is the call graph for this function:

size_t Chunk_NS::Chunk::getNumElemsForBytesInsertData	(	const DataBlockPtr &	src_data,
		const size_t	num_elems,
		const size_t	start_idx,
		const size_t	byte_limit,
		const bool	replicating = false
	)

Definition at line 125 of file Chunk.cpp.

References DataBlockPtr::arraysPtr, buffer, CHECK(), CHECK_EQ, column_desc, ColumnDescriptor::columnType, Data_Namespace::AbstractBuffer::encoder, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), StringNoneEncoder::getNumElemsForBytesInsertData(), FixedLengthArrayNoneEncoder::getNumElemsForBytesInsertData(), ArrayNoneEncoder::getNumElemsForBytesInsertData(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_varlen(), kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, and DataBlockPtr::stringsPtr.

                                                                    {
  CHECK(column_desc->columnType.is_varlen());
  switch (column_desc->columnType.get_type()) {
    case kARRAY: {
      if (column_desc->columnType.get_size() > 0) {
        FixedLengthArrayNoneEncoder* array_encoder =
            dynamic_cast<FixedLengthArrayNoneEncoder*>(buffer->encoder.get());
        return array_encoder->getNumElemsForBytesInsertData(
            src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating);
      }
      ArrayNoneEncoder* array_encoder =
          dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
      return array_encoder->getNumElemsForBytesInsertData(
          src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating);
    }
    case kTEXT:
    case kVARCHAR:
    case kCHAR: {
      CHECK_EQ(kENCODING_NONE, column_desc->columnType.get_compression());
      StringNoneEncoder* str_encoder =
          dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
      return str_encoder->getNumElemsForBytesInsertData(
          src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating);
    }
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON: {
      StringNoneEncoder* str_encoder =
          dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
      return str_encoder->getNumElemsForBytesInsertData(
          src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating);
    }
    default:
      CHECK(false);
      return 0;
  }
}

Here is the call graph for this function:

void Chunk_NS::Chunk::init_encoder

(

)

Definition at line 221 of file Chunk.cpp.

References buffer, CHECK(), CHECK_EQ, column_desc, ColumnDescriptor::columnType, Data_Namespace::AbstractBuffer::encoder, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), index_buf, Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_fixlen_array(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_varlen(), kARRAY, kCHAR, kENCODING_NONE, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTEXT, kVARCHAR, StringNoneEncoder::set_index_buf(), and ArrayNoneEncoder::set_index_buf().

                         {
  buffer->initEncoder(column_desc->columnType);
  if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) {
    switch (column_desc->columnType.get_type()) {
      case kARRAY: {
        ArrayNoneEncoder* array_encoder =
            dynamic_cast<ArrayNoneEncoder*>(buffer->encoder.get());
        array_encoder->set_index_buf(index_buf);
        break;
      }
      case kTEXT:
      case kVARCHAR:
      case kCHAR: {
        CHECK_EQ(kENCODING_NONE, column_desc->columnType.get_compression());
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
        str_encoder->set_index_buf(index_buf);
        break;
      }
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON: {
        StringNoneEncoder* str_encoder =
            dynamic_cast<StringNoneEncoder*>(buffer->encoder.get());
        str_encoder->set_index_buf(index_buf);
        break;
      }
      default:
        CHECK(false);
    }
  }
}

Here is the call graph for this function:

bool Chunk_NS::Chunk::isChunkOnDevice	(	DataMgr *	data_mgr,
		const ChunkKey &	key,
		const MemoryLevel	mem_level,
		const int	device_id
	)

Definition at line 40 of file Chunk.cpp.

References column_desc, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::is_fixlen_array(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_varlen(), and Data_Namespace::DataMgr::isBufferOnDevice().

                                                 {
  if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) {
    ChunkKey subKey = key;
    ChunkKey indexKey(subKey);
    indexKey.push_back(1);
    ChunkKey dataKey(subKey);
    dataKey.push_back(2);
    return data_mgr->isBufferOnDevice(indexKey, mem_level, device_id) &&
           data_mgr->isBufferOnDevice(dataKey, mem_level, device_id);
  } else {
    return data_mgr->isBufferOnDevice(key, mem_level, device_id);
  }
}

Here is the call graph for this function:

void Chunk_NS::Chunk::set_buffer ( AbstractBuffer *  b )

inline

Definition at line 96 of file Chunk.h.

References buffer.

{ buffer = b; }

void Chunk_NS::Chunk::set_index_buf ( AbstractBuffer *  ib )

inline

Definition at line 97 of file Chunk.h.

References index_buf.

{ index_buf = ib; }

static void Chunk_NS::Chunk::translateColumnDescriptorsToChunkVec ( const std::list< const ColumnDescriptor * > &  colDescs, std::vector< Chunk > &  chunkVec  )

inlinestatic

Definition at line 53 of file Chunk.h.

                                  {
    for (auto cd : colDescs) {
      chunkVec.emplace_back(cd);
    }
  }

void Chunk_NS::Chunk::unpin_buffer ( )

private

Definition at line 212 of file Chunk.cpp.

References buffer, index_buf, and Data_Namespace::AbstractBuffer::unPin().

Referenced by ~Chunk().

                         {
  if (buffer != nullptr) {
    buffer->unPin();
  }
  if (index_buf != nullptr) {
    index_buf->unPin();
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

AbstractBuffer* Chunk_NS::Chunk::buffer

private

Definition at line 102 of file Chunk.h.

Referenced by appendData(), begin_iterator(), createChunkBuffer(), get_buffer(), getChunkBuffer(), getNumElemsForBytesInsertData(), init_encoder(), set_buffer(), and unpin_buffer().

const ColumnDescriptor* Chunk_NS::Chunk::column_desc

private

Definition at line 104 of file Chunk.h.

Referenced by appendData(), begin_iterator(), createChunkBuffer(), get_column_desc(), getChunkBuffer(), getNumElemsForBytesInsertData(), init_encoder(), and isChunkOnDevice().

AbstractBuffer* Chunk_NS::Chunk::index_buf

private

Definition at line 103 of file Chunk.h.

Referenced by begin_iterator(), createChunkBuffer(), get_index_buf(), getChunkBuffer(), init_encoder(), set_index_buf(), and unpin_buffer().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/Chunk/Chunk.h
• /home/jenkins-slave/workspace/core-os-doxygen/Chunk/Chunk.cpp