Fragmenter_Namespace Namespace Reference

Namespaces
	anonymous_namespace{InsertOrderFragmenter.cpp}

Classes
class	AbstractFragmenter
struct	ArrayChunkConverter
struct	BlockWithColumnId
struct	ChunkToInsertDataConverter
struct	DateChunkConverter
struct	FixedLenArrayChunkConverter
class	FragmentInfo
	Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(rows) currently stored by that fragment. More...
struct	InsertData
	The data to be inserted using the fragment manager. More...
struct	InsertDataLoader
class	InsertOrderFragmenter
	The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter. More...
class	RowDataProvider
struct	ScalarChunkConverter
struct	ShardDataOwner
class	SortedOrderFragmenter
struct	StringChunkConverter
class	TableInfo

Enumerations
enum	FragmenterType { INSERT_ORDER = 0 }

Functions
template<typename SRC >
std::vector< std::vector< size_t > >	computeRowIndicesOfShards (size_t shardCount, size_t leafCount, size_t rowCount, SRC *src)
template<typename T >
size_t	indexOf (std::vector< T > &vec, T val)
bool	isStringVectorData (const ColumnDescriptor *cd)
bool	isDatumVectorData (const ColumnDescriptor *cd)
size_t	sizeOfRawColumn (const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd)
std::vector< std::vector< size_t > >	computeRowIndicesOfShards (const Catalog_Namespace::Catalog &cat, size_t leafCount, InsertData &insert_data)
template<typename T >
void	copyColumnDataOfShard (const std::vector< size_t > &rowIndices, T *src, T *dst)
BlockWithColumnId	copyColumnDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, const std::vector< size_t > &rowIndices, const ColumnDescriptor *pCol, size_t columnIndex, DataBlockPtr dataBlock)
InsertData	copyDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, InsertData &insert_data, int shardTableIndex, const std::vector< size_t > &rowIndices)
template<typename T >
void	shuffleByIndexesImpl (const std::vector< size_t > &indexes, T *buffer)
template<typename T >
void	shuffleByIndexesImpl (const std::vector< size_t > &indexes, std::vector< T > &buffer)
void	shuffleByIndexes (const ColumnDescriptor *cd, const std::vector< size_t > &indexes, DataBlockPtr &data)
template<typename T >
void	sortIndexesImpl (std::vector< size_t > &indexes, const T *buffer)
void	sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< std::string > &buffer)
void	sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< ArrayDatum > &buffer)
void	sortIndexes (const ColumnDescriptor *cd, std::vector< size_t > &indexes, const DataBlockPtr &data)
void	wait_cleanup_threads (std::vector< std::future< void >> &threads)
bool	is_integral (const SQLTypeInfo &t)
static int	get_chunks (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
template<typename T >
static void	set_chunk_stats (const SQLTypeInfo &col_type, int8_t *data_addr, int8_t &has_null, T &min, T &max)
static void	set_chunk_metadata (const Catalog_Namespace::Catalog *catalog, FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const size_t nrows_to_keep, UpdelRoll &updel_roll)

Enumeration Type Documentation

FragmenterType

enum Fragmenter_Namespace::FragmenterType

stores the type of a child class of AbstractTableFragmenter

Enumerator
INSERT_ORDER

Definition at line 44 of file Fragmenter.h.

                    {
  INSERT_ORDER = 0  // these values persist in catalog.  make explicit
};

Function Documentation

computeRowIndicesOfShards() [1/2]

template<typename SRC >

std::vector<std::vector<size_t> > Fragmenter_Namespace::computeRowIndicesOfShards	(	size_t	shardCount,
		size_t	leafCount,
		size_t	rowCount,
		SRC *	src
	)

Definition at line 34 of file InsertDataLoader.cpp.

References SHARD_FOR_KEY.

Referenced by computeRowIndicesOfShards(), and Fragmenter_Namespace::InsertDataLoader::insertData().

                                                                     {
  const auto numShardTables = shardCount * leafCount;

  std::vector<std::vector<size_t>> rowIndicesOfShards(numShardTables);

  for (size_t row = 0; row < rowCount; row++) {
    // expecting unsigned data
    // thus, no need for double remainder
    auto shardId = (std::is_unsigned<SRC>::value)
                       ? src[row] % numShardTables
                       : SHARD_FOR_KEY(src[row], numShardTables);
    rowIndicesOfShards[shardId].push_back(row);
  }

  return rowIndicesOfShards;
}

Here is the caller graph for this function:

computeRowIndicesOfShards() [2/2]

std::vector<std::vector<size_t> > Fragmenter_Namespace::computeRowIndicesOfShards	(	const Catalog_Namespace::Catalog &	cat,
		size_t	leafCount,
		InsertData &	insert_data
	)

Definition at line 109 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertData::columnIds, computeRowIndicesOfShards(), Fragmenter_Namespace::InsertData::data, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), indexOf(), isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::InsertData::numRows, sizeOfRawColumn(), and Fragmenter_Namespace::InsertData::tableId.

                             {
  const auto* td = cat.getMetadataForTable(insert_data.tableId);
  const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
  auto shardDataBlockIndex = indexOf(insert_data.columnIds, shard_cd->columnId);
  DataBlockPtr& shardDataBlock = insert_data.data[shardDataBlockIndex];
  auto rowCount = insert_data.numRows;
  auto shardCount = td->nShards;

  CHECK(!isStringVectorData(shard_cd));
  CHECK(!isDatumVectorData(shard_cd));

  switch (sizeOfRawColumn(cat, shard_cd)) {
    case 1:
      return computeRowIndicesOfShards(
          shardCount,
          leafCount,
          rowCount,
          reinterpret_cast<uint8_t*>(shardDataBlock.numbersPtr));
    case 2:
      return computeRowIndicesOfShards(
          shardCount,
          leafCount,
          rowCount,
          reinterpret_cast<uint16_t*>(shardDataBlock.numbersPtr));
    case 4:
      return computeRowIndicesOfShards(
          shardCount,
          leafCount,
          rowCount,
          reinterpret_cast<uint32_t*>(shardDataBlock.numbersPtr));
    case 8:
      return computeRowIndicesOfShards(
          shardCount,
          leafCount,
          rowCount,
          reinterpret_cast<uint64_t*>(shardDataBlock.numbersPtr));
  }
  throw std::runtime_error("Unexpected data block element size");
}

Here is the call graph for this function:

copyColumnDataOfShard() [1/2]

template<typename T >

void Fragmenter_Namespace::copyColumnDataOfShard	(	const std::vector< size_t > &	rowIndices,
		T *	src,
		T *	dst
	)

Definition at line 153 of file InsertDataLoader.cpp.

Referenced by copyColumnDataOfShard(), and copyDataOfShard().

                                                                                {
  for (size_t row = 0; row < rowIndices.size(); row++) {
    auto srcRowIndex = rowIndices[row];
    dst[row] = src[srcRowIndex];
  }
}

Here is the caller graph for this function:

copyColumnDataOfShard() [2/2]

BlockWithColumnId Fragmenter_Namespace::copyColumnDataOfShard	(	const Catalog_Namespace::Catalog &	cat,
		ShardDataOwner &	dataOwner,
		const std::vector< size_t > &	rowIndices,
		const ColumnDescriptor *	pCol,
		size_t	columnIndex,
		DataBlockPtr	dataBlock
	)

Definition at line 165 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, DataBlockPtr::arraysPtr, ColumnDescriptor::columnId, copyColumnDataOfShard(), isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::ShardDataOwner::rawData, sizeOfRawColumn(), Fragmenter_Namespace::ShardDataOwner::stringData, and DataBlockPtr::stringsPtr.

                                                                {
  DataBlockPtr ret;
  if (isStringVectorData(pCol)) {
    auto& data = dataOwner.stringData[columnIndex];
    data.resize(rowIndices.size());
    copyColumnDataOfShard(rowIndices, &(*(dataBlock.stringsPtr))[0], &data[0]);
    ret.stringsPtr = &data;

  } else if (isDatumVectorData(pCol)) {
    auto& data = dataOwner.arrayData[columnIndex];
    data.resize(rowIndices.size());
    copyColumnDataOfShard(rowIndices, &(*(dataBlock.arraysPtr))[0], &data[0]);
    ret.arraysPtr = &data;

  } else {
    auto rawArrayElementSize = sizeOfRawColumn(cat, pCol);
    auto& data = dataOwner.rawData[columnIndex];
    data.resize(rowIndices.size() * rawArrayElementSize);

    switch (rawArrayElementSize) {
      case 1: {
        copyColumnDataOfShard(rowIndices,
                              reinterpret_cast<uint8_t*>(dataBlock.numbersPtr),
                              reinterpret_cast<uint8_t*>(&data[0]));
        break;
      }
      case 2: {
        copyColumnDataOfShard(rowIndices,
                              reinterpret_cast<uint16_t*>(dataBlock.numbersPtr),
                              reinterpret_cast<uint16_t*>(&data[0]));
        break;
      }
      case 4: {
        copyColumnDataOfShard(rowIndices,
                              reinterpret_cast<uint32_t*>(dataBlock.numbersPtr),
                              reinterpret_cast<uint32_t*>(&data[0]));
        break;
      }
      case 8: {
        copyColumnDataOfShard(rowIndices,
                              reinterpret_cast<uint64_t*>(dataBlock.numbersPtr),
                              reinterpret_cast<uint64_t*>(&data[0]));
        break;
      }
      default:
        throw std::runtime_error("Unexpected data block element size");
    }

    ret.numbersPtr = reinterpret_cast<int8_t*>(&data[0]);
  }

  return {pCol->columnId, ret};
}

Here is the call graph for this function:

copyDataOfShard()

InsertData Fragmenter_Namespace::copyDataOfShard	(	const Catalog_Namespace::Catalog &	cat,
		ShardDataOwner &	dataOwner,
		InsertData &	insert_data,
		int	shardTableIndex,
		const std::vector< size_t > &	rowIndices
	)

Definition at line 224 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, cat(), Fragmenter_Namespace::InsertData::columnIds, copyColumnDataOfShard(), Fragmenter_Namespace::InsertData::data, Fragmenter_Namespace::InsertData::databaseId, Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getPhysicalTablesDescriptors(), indexOf(), Fragmenter_Namespace::InsertData::numRows, Fragmenter_Namespace::ShardDataOwner::rawData, Fragmenter_Namespace::ShardDataOwner::stringData, and Fragmenter_Namespace::InsertData::tableId.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

                                                                {
  const auto* td = cat.getMetadataForTable(insert_data.tableId);
  const auto* ptd = cat.getPhysicalTablesDescriptors(td)[shardTableIndex];

  InsertData shardData;
  shardData.databaseId = insert_data.databaseId;
  shardData.tableId = ptd->tableId;
  shardData.numRows = rowIndices.size();

  std::vector<const ColumnDescriptor*> pCols;
  std::vector<int> lCols;

  {
    auto logicalColumns = cat.getAllColumnMetadataForTable(td->tableId, true, true, true);
    for (const auto& cd : logicalColumns) {
      lCols.push_back(cd->columnId);
    }

    auto physicalColumns =
        cat.getAllColumnMetadataForTable(ptd->tableId, true, true, true);
    for (const auto& cd : physicalColumns) {
      pCols.push_back(cd);
    }
  }

  for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
    dataOwner.arrayData.emplace_back();
    dataOwner.rawData.emplace_back();
    dataOwner.stringData.emplace_back();
  }

  auto copycat = [&cat, &dataOwner, &rowIndices, &lCols, &pCols, &insert_data](int col) {
    const auto lColId = insert_data.columnIds[col];
    const auto pCol = pCols[indexOf(lCols, lColId)];
    return copyColumnDataOfShard(
        cat, dataOwner, rowIndices, pCol, col, insert_data.data[col]);
  };

  std::vector<std::future<BlockWithColumnId>> worker_threads;
  for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
    worker_threads.push_back(std::async(std::launch::async, copycat, col));
  }

  for (auto& child : worker_threads) {
    child.wait();
  }

  for (auto& child : worker_threads) {
    auto shardColumnData = child.get();
    shardData.columnIds.push_back(shardColumnData.columnId);
    shardData.data.push_back(shardColumnData.block);
  }

  return shardData;
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_chunks()

static int Fragmenter_Namespace::get_chunks ( const Catalog_Namespace::Catalog *  catalog, const TableDescriptor *  td, const FragmentInfo &  fragment, const Data_Namespace::MemoryLevel  memory_level, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &  chunks  )

static

Definition at line 74 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::DBMetadata::dbId, Fragmenter_Namespace::FragmentInfo::fragmentId, Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), TableDescriptor::nColumns, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

                                                                       {
  for (int cid = 1, nc = 0; nc < td->nColumns; ++cid) {
    if (const auto cd = catalog->getMetadataForColumn(td->tableId, cid)) {
      ++nc;
      if (!cd->isVirtualCol) {
        auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cid);
        CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
        ChunkKey chunk_key{
            catalog->getCurrentDB().dbId, td->tableId, cid, fragment.fragmentId};
        auto chunk = Chunk_NS::Chunk::getChunk(cd,
                                               &catalog->getDataMgr(),
                                               chunk_key,
                                               memory_level,
                                               0,
                                               chunk_meta_it->second->numBytes,
                                               chunk_meta_it->second->numElements);
        chunks.push_back(chunk);
      }
    }
  }
  return chunks.size();
}

Here is the call graph for this function:

Here is the caller graph for this function:

indexOf()

template<typename T >

size_t Fragmenter_Namespace::indexOf	(	std::vector< T > &	vec,
		T	val
	)

Definition at line 55 of file InsertDataLoader.cpp.

References CHECK.

Referenced by computeRowIndicesOfShards(), and copyDataOfShard().

                                         {
  typename std::vector<T>::iterator it = std::find(vec.begin(), vec.end(), val);
  CHECK(it != vec.end());
  return std::distance(vec.begin(), it);
}

Here is the caller graph for this function:

is_integral()

bool Fragmenter_Namespace::is_integral ( const SQLTypeInfo &  t )

inline

Definition at line 48 of file UpdelStorage.cpp.

References SQLTypeInfo::is_boolean(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_time(), SQLTypeInfo::is_timeinterval(), and Fragmenter_Namespace::FragmentInfo::unconditionalVacuum_.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata().

                                              {
  return t.is_integer() || t.is_boolean() || t.is_time() || t.is_timeinterval();
}

Here is the call graph for this function:

Here is the caller graph for this function:

isDatumVectorData()

bool Fragmenter_Namespace::isDatumVectorData

(

const ColumnDescriptor *

cd

)

Definition at line 67 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::is_array().

Referenced by computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                   {
  return cd->columnType.is_array();
}

Here is the call graph for this function:

Here is the caller graph for this function:

isStringVectorData()

bool Fragmenter_Namespace::isStringVectorData

(

const ColumnDescriptor *

cd

)

Definition at line 61 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::is_geometry(), SQLTypeInfo::is_string(), and kENCODING_NONE.

Referenced by computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                    {
  return (cd->columnType.is_geometry()) ||
         (cd->columnType.is_string() &&
          cd->columnType.get_compression() == kENCODING_NONE);
}

Here is the call graph for this function:

Here is the caller graph for this function:

set_chunk_metadata()

static void Fragmenter_Namespace::set_chunk_metadata ( const Catalog_Namespace::Catalog *  catalog, FragmentInfo &  fragment, const std::shared_ptr< Chunk_NS::Chunk > &  chunk, const size_t  nrows_to_keep, UpdelRoll &  updel_roll  )

static

Definition at line 1046 of file UpdelStorage.cpp.

References UpdelRoll::chunkMetadata, UpdelRoll::dirtyChunks, Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getMetadataForTable(), and UpdelRoll::mutex.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

                                                      {
  auto cd = chunk->getColumnDesc();
  auto td = catalog->getMetadataForTable(cd->tableId);
  auto data_buffer = chunk->getBuffer();
  std::lock_guard<std::mutex> lck(updel_roll.mutex);
  const auto key = std::make_pair(td, &fragment);
  if (0 == updel_roll.chunkMetadata.count(key)) {
    updel_roll.chunkMetadata[key] = fragment.getChunkMetadataMapPhysical();
  }
  auto& chunkMetadata = updel_roll.chunkMetadata[key];
  chunkMetadata[cd->columnId]->numElements = nrows_to_keep;
  chunkMetadata[cd->columnId]->numBytes = data_buffer->size();
  if (updel_roll.dirtyChunks.count(chunk.get()) == 0) {
    updel_roll.dirtyChunks.emplace(chunk.get(), chunk);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

set_chunk_stats()

template<typename T >

static void Fragmenter_Namespace::set_chunk_stats ( const SQLTypeInfo &  col_type, int8_t *  data_addr, int8_t &  has_null, T &  min, T &  max  )

static

Definition at line 1031 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), is_null(), anonymous_namespace{TypedDataAccessors.h}::set_minmax(), and omnisci.dtypes::T.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

                                    {
  T v;
  const auto can_be_null = !col_type.get_notnull();
  const auto is_null = get_scalar<T>(data_addr, col_type, v);
  if (is_null) {
    has_null = has_null || (can_be_null && is_null);
  } else {
    set_minmax(min, max, v);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

shuffleByIndexes()

void Fragmenter_Namespace::shuffleByIndexes	(	const ColumnDescriptor *	cd,
		const std::vector< size_t > &	indexes,
		DataBlockPtr &	data
	)

Definition at line 44 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, shuffleByIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

                                          {
  const auto& ti = cd->columnType;
  switch (ti.get_type()) {
    case kBOOLEAN:
      shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
      break;
    case kTINYINT:
      shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
      break;
    case kSMALLINT:
      shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
      break;
    case kINT:
      shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
      break;
    case kBIGINT:
    case kNUMERIC:
    case kDECIMAL:
      shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
      break;
    case kFLOAT:
      shuffleByIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
      break;
    case kDOUBLE:
      shuffleByIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
      break;
    case kTEXT:
    case kVARCHAR:
    case kCHAR:
      if (ti.is_varlen()) {
        shuffleByIndexesImpl(indexes, *data.stringsPtr);
      } else {
        switch (ti.get_size()) {
          case 1:
            shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
            break;
          case 2:
            shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
            break;
          case 4:
            shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
            break;
          default:
            CHECK(false);
        }
      }
      break;
    case kDATE:
    case kTIME:
    case kTIMESTAMP:
      shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
      break;
    case kARRAY:
      shuffleByIndexesImpl(indexes, *data.arraysPtr);
      break;
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON:
      shuffleByIndexesImpl(indexes, *data.stringsPtr);
      break;
    default:
      CHECK(false);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

shuffleByIndexesImpl() [1/2]

template<typename T >

void Fragmenter_Namespace::shuffleByIndexesImpl	(	const std::vector< size_t > &	indexes,
		T *	buffer
	)

Definition at line 25 of file SortedOrderFragmenter.cpp.

References omnisci.dtypes::T.

Referenced by shuffleByIndexes().

                                                                       {
  std::vector<T> new_buffer;
  new_buffer.reserve(indexes.size());
  for (const auto i : indexes) {
    new_buffer.push_back(buffer[i]);
  }
  std::memcpy(buffer, new_buffer.data(), indexes.size() * sizeof(T));
}

Here is the caller graph for this function:

shuffleByIndexesImpl() [2/2]

template<typename T >

void Fragmenter_Namespace::shuffleByIndexesImpl	(	const std::vector< size_t > &	indexes,
		std::vector< T > &	buffer
	)

Definition at line 35 of file SortedOrderFragmenter.cpp.

                                                                                  {
  std::vector<T> new_buffer;
  new_buffer.reserve(indexes.size());
  for (const auto i : indexes) {
    new_buffer.push_back(buffer[i]);
  }
  buffer.swap(new_buffer);
}

sizeOfRawColumn()

size_t Fragmenter_Namespace::sizeOfRawColumn	(	const Catalog_Namespace::Catalog &	cat,
		const ColumnDescriptor *	cd
	)

Definition at line 71 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_logical_size(), SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_NONE, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

Referenced by computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                   {
  switch (cd->columnType.get_type()) {
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON:
    case kARRAY:
      throw std::runtime_error("geo and array columns have variable length elements");
    case kBOOLEAN:
    case kTINYINT:
    case kSMALLINT:
    case kINT:
    case kBIGINT:
    case kNUMERIC:
    case kDECIMAL:
    case kFLOAT:
    case kDOUBLE:
    case kTIMESTAMP:
    case kTIME:
    case kINTERVAL_DAY_TIME:
    case kINTERVAL_YEAR_MONTH:
    case kDATE:
      return cd->columnType.get_logical_size();
    case kTEXT:
    case kVARCHAR:
    case kCHAR:
      if (cd->columnType.get_compression() == kENCODING_NONE) {
        throw std::runtime_error(
            "non encoded string columns have variable length elements");
      }
      return cd->columnType.get_size();
    default:
      throw std::runtime_error("not supported column type: " + cd->columnName + " (" +
                               cd->columnType.get_type_name() + ")");
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

sortIndexes()

void Fragmenter_Namespace::sortIndexes	(	const ColumnDescriptor *	cd,
		std::vector< size_t > &	indexes,
		const DataBlockPtr &	data
	)

Definition at line 137 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, sortIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

                                           {
  const auto& ti = cd->columnType;
  switch (ti.get_type()) {
    case kBOOLEAN:
      sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
      break;
    case kTINYINT:
      sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
      break;
    case kSMALLINT:
      sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
      break;
    case kINT:
      sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
      break;
    case kBIGINT:
    case kNUMERIC:
    case kDECIMAL:
      sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
      break;
    case kFLOAT:
      sortIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
      break;
    case kDOUBLE:
      sortIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
      break;
    case kTEXT:
    case kVARCHAR:
    case kCHAR:
      if (ti.is_varlen()) {
        sortIndexesImpl(indexes, *data.stringsPtr);
      } else {
        switch (ti.get_size()) {
          case 1:
            sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
            break;
          case 2:
            sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
            break;
          case 4:
            sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
            break;
          default:
            CHECK(false);
        }
      }
      break;
    case kDATE:
    case kTIME:
    case kTIMESTAMP:
      sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
      break;
    case kARRAY:
      sortIndexesImpl(indexes, *data.arraysPtr);
      break;
    default:
      CHECK(false) << "invalid type '" << ti.get_type() << "' to sort";
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

sortIndexesImpl() [1/3]

template<typename T >

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const T *	buffer
	)

Definition at line 113 of file SortedOrderFragmenter.cpp.

References CHECK, and gpu_enabled::sort().

Referenced by sortIndexes().

                                                                  {
  CHECK(buffer);
  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
    return buffer[a] < buffer[b];
  });
}

Here is the call graph for this function:

Here is the caller graph for this function:

sortIndexesImpl() [2/3]

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const std::vector< std::string > &	buffer
	)

Definition at line 120 of file SortedOrderFragmenter.cpp.

References gpu_enabled::sort().

                                                         {
  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
    return buffer[a].size() < buffer[b].size() ||
           (buffer[a].size() == buffer[b].size() && buffer[a] < buffer[b]);
  });
}

Here is the call graph for this function:

sortIndexesImpl() [3/3]

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const std::vector< ArrayDatum > &	buffer
	)

Definition at line 128 of file SortedOrderFragmenter.cpp.

References gpu_enabled::sort().

                                                          {
  std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
    return buffer[a].is_null || buffer[a].length < buffer[b].length ||
           (!buffer[b].is_null && buffer[a].length == buffer[b].length &&
            memcmp(buffer[a].pointer, buffer[b].pointer, buffer[a].length) < 0);
  });
}

Here is the call graph for this function:

wait_cleanup_threads()

void Fragmenter_Namespace::wait_cleanup_threads ( std::vector< std::future< void >> &  threads )

inline

Definition at line 41 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows(), Fragmenter_Namespace::InsertOrderFragmenter::getVacuumOffsets(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

                                                                      {
  for (auto& t : threads) {
    t.get();
  }
  threads.clear();
}

Here is the caller graph for this function: