Namespaces
	anonymous_namespace{InsertOrderFragmenter.cpp}

	anonymous_namespace{UpdelStorage.cpp}

Classes
class	RowDataProvider

struct	UpdateValuesStats

struct	ChunkUpdateStats

class	AbstractFragmenter

struct	InsertData
	The data to be inserted using the fragment manager. More...

class	FragmentInfo
	Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(rows) currently stored by that fragment. More...

class	TableInfo

struct	ShardDataOwner

struct	BlockWithColumnId

struct	InsertDataLoader

class	InsertOrderFragmenter
	The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter. More...

class	SortedOrderFragmenter

struct	ChunkToInsertDataConverter

struct	ScalarChunkConverter

struct	FixedLenArrayChunkConverter

struct	ArrayChunkConverter

struct	StringChunkConverter

struct	DateChunkConverter

Enumerations
enum	FragmenterType { INSERT_ORDER = 0 }

Functions
template<typename SRC >
std::vector< std::vector < size_t > >	computeRowIndicesOfShards (size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)

template<typename T >
size_t	indexOf (std::vector< T > &vec, T val)

bool	isStringVectorData (const ColumnDescriptor *cd)

bool	isDatumVectorData (const ColumnDescriptor *cd)

size_t	sizeOfRawColumn (const Catalog_Namespace::Catalog &cat, const ColumnDescriptor *cd)

std::vector< std::vector < size_t > >	computeRowIndicesOfShards (const Catalog_Namespace::Catalog &cat, size_t leafCount, InsertData &insert_data)

template<typename T >
void	copyColumnDataOfShard (const std::vector< size_t > &rowIndices, T src, T dst)

BlockWithColumnId	copyColumnDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, const std::vector< size_t > &rowIndices, const ColumnDescriptor *pCol, size_t columnIndex, DataBlockPtr dataBlock, bool is_default)

InsertData	copyDataOfShard (const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, InsertData &insert_data, int shardTableIndex, const std::vector< size_t > &rowIndices)

template<typename T >
void	shuffleByIndexesImpl (const std::vector< size_t > &indexes, T *buffer)

template<typename T >
void	shuffleByIndexesImpl (const std::vector< size_t > &indexes, std::vector< T > &buffer)

void	shuffleByIndexes (const ColumnDescriptor *cd, const std::vector< size_t > &indexes, DataBlockPtr &data)

template<typename T >
void	sortIndexesImpl (std::vector< size_t > &indexes, const T *buffer)

void	sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< std::string > &buffer)

void	sortIndexesImpl (std::vector< size_t > &indexes, const std::vector< ArrayDatum > &buffer)

void	sortIndexes (const ColumnDescriptor *cd, std::vector< size_t > &indexes, const DataBlockPtr &data)

void	wait_cleanup_threads (std::vector< std::future< void >> &threads)

bool	is_integral (const SQLTypeInfo &t)

static int	get_chunks (const Catalog_Namespace::Catalog catalog, const TableDescriptor td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks)

template<typename T >
static void	set_chunk_stats (const SQLTypeInfo &col_type, int8_t *data_addr, bool &has_null, T &min, T &max)

static void	set_chunk_metadata (const Catalog_Namespace::Catalog *catalog, FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const size_t nrows_to_keep, UpdelRoll &updel_roll)

size_t	get_null_padding (bool is_varlen_array, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)

std::set< size_t >	get_var_len_null_array_indexes (const SQLTypeInfo sql_type_info, const std::vector< uint64_t > &frag_offsets, const StringOffsetT *index_array, size_t fragment_row_count)

StringOffsetT	get_buffer_offset (bool is_varlen_array, const StringOffsetT *index_array, size_t index)

Enumeration Type Documentation

enum Fragmenter_Namespace::FragmenterType

stores the type of a child class of AbstractTableFragmenter

Enumerator
INSERT_ORDER

Definition at line 44 of file Fragmenter.h.

                     {
   INSERT_ORDER = 0  // these values persist in catalog.  make explicit
 };

Function Documentation

template<typename SRC >

std::vector<std::vector<size_t> > Fragmenter_Namespace::computeRowIndicesOfShards	(	size_t	shard_count,
		size_t	leaf_count,
		size_t	row_count,
		SRC *	src,
		bool	duplicated_key_value
	)

Definition at line 35 of file InsertDataLoader.cpp.

References SHARD_FOR_KEY.

Referenced by computeRowIndicesOfShards(), and Fragmenter_Namespace::InsertDataLoader::insertData().

                                                                                       {
   const auto n_shard_tables = shard_count * leaf_count;
   std::vector<std::vector<size_t>> row_indices_of_shards(n_shard_tables);
   if (!duplicated_key_value) {
     for (size_t row = 0; row < row_count; row++) {
       // expecting unsigned data
       // thus, no need for double remainder
       auto shard_id = (std::is_unsigned<SRC>::value)
                           ? src[row] % n_shard_tables
                           : SHARD_FOR_KEY(src[row], n_shard_tables);
       row_indices_of_shards[shard_id].push_back(row);
     }
   } else {
     auto shard_id = (std::is_unsigned<SRC>::value)
                         ? src[0] % n_shard_tables
                         : SHARD_FOR_KEY(src[0], n_shard_tables);
     row_indices_of_shards[shard_id].reserve(row_count);
     for (size_t row = 0; row < row_count; row++) {
       row_indices_of_shards[shard_id].push_back(row);
     }
   }
 
   return row_indices_of_shards;
 }

Here is the caller graph for this function:

std::vector<std::vector<size_t> > Fragmenter_Namespace::computeRowIndicesOfShards	(	const Catalog_Namespace::Catalog &	cat,
		size_t	leafCount,
		InsertData &	insert_data
	)

Definition at line 119 of file InsertDataLoader.cpp.

References CHECK, Fragmenter_Namespace::InsertData::columnIds, computeRowIndicesOfShards(), Fragmenter_Namespace::InsertData::data, Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getShardColumnMetadataForTable(), indexOf(), Fragmenter_Namespace::InsertData::is_default, isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::InsertData::numRows, sizeOfRawColumn(), and Fragmenter_Namespace::InsertData::tableId.

                              {
   const auto* td = cat.getMetadataForTable(insert_data.tableId);
   const auto* shard_cd = cat.getShardColumnMetadataForTable(td);
   auto shardDataBlockIndex = indexOf(insert_data.columnIds, shard_cd->columnId);
   DataBlockPtr& shardDataBlock = insert_data.data[shardDataBlockIndex];
   auto rowCount = insert_data.numRows;
   auto shardCount = td->nShards;
 
   CHECK(!isStringVectorData(shard_cd));
   CHECK(!isDatumVectorData(shard_cd));
 
   CHECK(insert_data.is_default.size() == insert_data.columnIds.size());
   bool is_default = insert_data.is_default[shardDataBlockIndex];
   switch (sizeOfRawColumn(cat, shard_cd)) {
     case 1:
       return computeRowIndicesOfShards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint8_t*>(shardDataBlock.numbersPtr),
           is_default);
     case 2:
       return computeRowIndicesOfShards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint16_t*>(shardDataBlock.numbersPtr),
           is_default);
     case 4:
       return computeRowIndicesOfShards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint32_t*>(shardDataBlock.numbersPtr),
           is_default);
     case 8:
       return computeRowIndicesOfShards(
           shardCount,
           leafCount,
           rowCount,
           reinterpret_cast<uint64_t*>(shardDataBlock.numbersPtr),
           is_default);
   }
   throw std::runtime_error("Unexpected data block element size");
 }

Here is the call graph for this function:

template<typename T >

void Fragmenter_Namespace::copyColumnDataOfShard	(	const std::vector< size_t > &	rowIndices,
		T *	src,
		T *	dst
	)

Definition at line 169 of file InsertDataLoader.cpp.

Referenced by copyColumnDataOfShard(), and copyDataOfShard().

                                                                                 {
   for (size_t row = 0; row < rowIndices.size(); row++) {
     auto srcRowIndex = rowIndices[row];
     dst[row] = src[srcRowIndex];
   }
 }

Here is the caller graph for this function:

BlockWithColumnId Fragmenter_Namespace::copyColumnDataOfShard	(	const Catalog_Namespace::Catalog &	cat,
		ShardDataOwner &	dataOwner,
		const std::vector< size_t > &	rowIndices,
		const ColumnDescriptor *	pCol,
		size_t	columnIndex,
		DataBlockPtr	dataBlock,
		bool	is_default
	)

Definition at line 182 of file InsertDataLoader.cpp.

References Fragmenter_Namespace::ShardDataOwner::arrayData, DataBlockPtr::arraysPtr, ColumnDescriptor::columnId, copyColumnDataOfShard(), isDatumVectorData(), isStringVectorData(), DataBlockPtr::numbersPtr, Fragmenter_Namespace::ShardDataOwner::rawData, sizeOfRawColumn(), Fragmenter_Namespace::ShardDataOwner::stringData, and DataBlockPtr::stringsPtr.

                                                          {
   DataBlockPtr ret;
   std::vector<size_t> single_row_idx({0ul});
   const std::vector<size_t>& rows = is_default ? single_row_idx : rowIndices;
   if (isStringVectorData(pCol)) {
     auto& data = dataOwner.stringData[columnIndex];
     data.resize(rows.size());
     copyColumnDataOfShard(rows, &(*(dataBlock.stringsPtr))[0], &data[0]);
     ret.stringsPtr = &data;
 
   } else if (isDatumVectorData(pCol)) {
     auto& data = dataOwner.arrayData[columnIndex];
     data.resize(rows.size());
     copyColumnDataOfShard(rows, &(*(dataBlock.arraysPtr))[0], &data[0]);
     ret.arraysPtr = &data;
 
   } else {
     auto rawArrayElementSize = sizeOfRawColumn(cat, pCol);
     auto& data = dataOwner.rawData[columnIndex];
     data.resize(rows.size() * rawArrayElementSize);
 
     switch (rawArrayElementSize) {
       case 1: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint8_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint8_t*>(&data[0]));
         break;
       }
       case 2: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint16_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint16_t*>(&data[0]));
         break;
       }
       case 4: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint32_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint32_t*>(&data[0]));
         break;
       }
       case 8: {
         copyColumnDataOfShard(rows,
                               reinterpret_cast<uint64_t*>(dataBlock.numbersPtr),
                               reinterpret_cast<uint64_t*>(&data[0]));
         break;
       }
       default:
         throw std::runtime_error("Unexpected data block element size");
     }
 
     ret.numbersPtr = reinterpret_cast<int8_t*>(&data[0]);
   }
 
   return {pCol->columnId, ret, is_default};
 }

Here is the call graph for this function:

InsertData Fragmenter_Namespace::copyDataOfShard	(	const Catalog_Namespace::Catalog &	cat,
		ShardDataOwner &	dataOwner,
		InsertData &	insert_data,
		int	shardTableIndex,
		const std::vector< size_t > &	rowIndices
	)

Definition at line 244 of file InsertDataLoader.cpp.

Referenced by Fragmenter_Namespace::InsertDataLoader::insertData().

                                                                 {
   const auto* td = cat.getMetadataForTable(insert_data.tableId);
   const auto* ptd = cat.getPhysicalTablesDescriptors(td)[shardTableIndex];
 
   InsertData shardData;
   shardData.databaseId = insert_data.databaseId;
   shardData.tableId = ptd->tableId;
   shardData.numRows = rowIndices.size();
 
   std::vector<const ColumnDescriptor*> pCols;
   std::vector<int> lCols;
 
   {
     auto logicalColumns = cat.getAllColumnMetadataForTable(td->tableId, true, true, true);
     for (const auto& cd : logicalColumns) {
       lCols.push_back(cd->columnId);
     }
 
     auto physicalColumns =
         cat.getAllColumnMetadataForTable(ptd->tableId, true, true, true);
     for (const auto& cd : physicalColumns) {
       pCols.push_back(cd);
     }
   }
 
   for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
     dataOwner.arrayData.emplace_back();
     dataOwner.rawData.emplace_back();
     dataOwner.stringData.emplace_back();
   }
 
   auto copycat = [&cat, &dataOwner, &rowIndices, &lCols, &pCols, &insert_data](int col) {
     const auto lColId = insert_data.columnIds[col];
     const auto pCol = pCols[indexOf(lCols, lColId)];
     return copyColumnDataOfShard(cat,
                                  dataOwner,
                                  rowIndices,
                                  pCol,
                                  col,
                                  insert_data.data[col],
                                  insert_data.is_default[col]);
   };
 
   std::vector<std::future<BlockWithColumnId>> worker_threads;
   for (size_t col = 0; col < insert_data.columnIds.size(); col++) {
     worker_threads.push_back(std::async(std::launch::async, copycat, col));
   }
 
   for (auto& child : worker_threads) {
     child.wait();
   }
 
   for (auto& child : worker_threads) {
     auto shardColumnData = child.get();
     shardData.columnIds.push_back(shardColumnData.columnId);
     shardData.data.push_back(shardColumnData.block);
     shardData.is_default.push_back(shardColumnData.is_default);
   }
 
   return shardData;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

StringOffsetT Fragmenter_Namespace::get_buffer_offset	(	bool	is_varlen_array,
		const StringOffsetT *	index_array,
		size_t	index
	)

Definition at line 1210 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

                                               {
   auto offset = index_array[index];
   if (offset < 0) {
     // Variable length arrays encode null arrays as negative offsets
     CHECK(is_varlen_array);
     offset = -offset;
   }
   return offset;
 }

Here is the caller graph for this function:

static int Fragmenter_Namespace::get_chunks	(	const Catalog_Namespace::Catalog *	catalog,
		const TableDescriptor *	td,
		const FragmentInfo &	fragment,
		const Data_Namespace::MemoryLevel	memory_level,
		std::vector< std::shared_ptr< Chunk_NS::Chunk >> &	chunks
	)

static

Definition at line 74 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::DBMetadata::dbId, Fragmenter_Namespace::FragmentInfo::fragmentId, Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), TableDescriptor::nColumns, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

                                                                        {
   for (int cid = 1, nc = 0; nc < td->nColumns; ++cid) {
     if (const auto cd = catalog->getMetadataForColumn(td->tableId, cid)) {
       ++nc;
       if (!cd->isVirtualCol) {
         auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cid);
         CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
         ChunkKey chunk_key{
             catalog->getCurrentDB().dbId, td->tableId, cid, fragment.fragmentId};
         auto chunk = Chunk_NS::Chunk::getChunk(cd,
                                                &catalog->getDataMgr(),
                                                chunk_key,
                                                memory_level,
                                                0,
                                                chunk_meta_it->second->numBytes,
                                                chunk_meta_it->second->numElements);
         chunks.push_back(chunk);
       }
     }
   }
   return chunks.size();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t Fragmenter_Namespace::get_null_padding	(	bool	is_varlen_array,
		const std::vector< uint64_t > &	frag_offsets,
		const StringOffsetT *	index_array,
		size_t	fragment_row_count
	)

Definition at line 1147 of file UpdelStorage.cpp.

References CHECK_GT, CHECK_LT, ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE, and i.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

                                                    {
   if (is_varlen_array) {
     size_t first_non_deleted_row_index{0};
     for (auto deleted_offset : frag_offsets) {
       if (first_non_deleted_row_index < deleted_offset) {
         break;
       } else {
         first_non_deleted_row_index++;
       }
     }
     CHECK_LT(first_non_deleted_row_index, fragment_row_count);
     if (first_non_deleted_row_index == 0) {
       // If the first row in the fragment is not deleted, then the first offset in the
       // index buffer/array already contains expected padding.
       return index_array[0];
     } else {
       // If the first non-deleted element is a null array (indentified by a negative
       // offset), get a padding value for the chunk buffer.
       if (index_array[first_non_deleted_row_index + 1] < 0) {
         size_t first_non_zero_offset{0};
         for (size_t i = 0; i <= first_non_deleted_row_index; i++) {
           if (index_array[i] != 0) {
             first_non_zero_offset = index_array[i];
             break;
           }
         }
         CHECK_GT(first_non_zero_offset, static_cast<size_t>(0));
         return std::min(ArrayNoneEncoder::DEFAULT_NULL_PADDING_SIZE,
                         first_non_zero_offset);
       } else {
         return 0;
       }
     }
   } else {
     return 0;
   }
 }

Here is the caller graph for this function:

std::set<size_t> Fragmenter_Namespace::get_var_len_null_array_indexes	(	const SQLTypeInfo	sql_type_info,
		const std::vector< uint64_t > &	frag_offsets,
		const StringOffsetT *	index_array,
		size_t	fragment_row_count
	)

Definition at line 1189 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), i, and SQLTypeInfo::is_varlen_array().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::vacuum_varlen_rows().

                                                                            {
   std::set<size_t> null_array_indexes;
   if (sql_type_info.is_varlen_array() && !sql_type_info.get_notnull()) {
     size_t frag_offset_index{0};
     size_t vacuum_offset{0};
     for (size_t i = 0; i < fragment_row_count; i++) {
       if (frag_offset_index < frag_offsets.size() &&
           i == frag_offsets[frag_offset_index]) {
         frag_offset_index++;
         vacuum_offset++;
       } else if (index_array[i + 1] < 0) {
         null_array_indexes.emplace(i - vacuum_offset);
       }
     }
   }
   return null_array_indexes;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

size_t Fragmenter_Namespace::indexOf	(	std::vector< T > &	vec,
		T	val
	)

Definition at line 65 of file InsertDataLoader.cpp.

References CHECK.

Referenced by org.apache.calcite.sql2rel.SqlToRelConverter::collectInsertTargets(), computeRowIndicesOfShards(), copyDataOfShard(), and com.mapd.utility.db_vendors.PostGis_types::get_wkt().

                                          {
   typename std::vector<T>::iterator it = std::find(vec.begin(), vec.end(), val);
   CHECK(it != vec.end());
   return std::distance(vec.begin(), it);
 }

Here is the caller graph for this function:

bool Fragmenter_Namespace::is_integral ( const SQLTypeInfo & t )

inline

Definition at line 48 of file UpdelStorage.cpp.

References SQLTypeInfo::is_boolean(), SQLTypeInfo::is_integer(), SQLTypeInfo::is_time(), and SQLTypeInfo::is_timeinterval().

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata().

                                               {
   return t.is_integer() || t.is_boolean() || t.is_time() || t.is_timeinterval();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool Fragmenter_Namespace::isDatumVectorData ( const ColumnDescriptor * cd )

Definition at line 77 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, and SQLTypeInfo::is_array().

Referenced by computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                    {
   return cd->columnType.is_array();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool Fragmenter_Namespace::isStringVectorData ( const ColumnDescriptor * cd )

Definition at line 71 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::is_geometry(), SQLTypeInfo::is_string(), and kENCODING_NONE.

Referenced by computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                     {
   return (cd->columnType.is_geometry()) ||
          (cd->columnType.is_string() &&
           cd->columnType.get_compression() == kENCODING_NONE);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

static void Fragmenter_Namespace::set_chunk_metadata	(	const Catalog_Namespace::Catalog *	catalog,
		FragmentInfo &	fragment,
		const std::shared_ptr< Chunk_NS::Chunk > &	chunk,
		const size_t	nrows_to_keep,
		UpdelRoll &	updel_roll
	)

static

Definition at line 1084 of file UpdelStorage.cpp.

References UpdelRoll::chunkMetadata, UpdelRoll::dirtyChunkeys, UpdelRoll::dirtyChunks, Fragmenter_Namespace::FragmentInfo::fragmentId, Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getMetadataForTable(), and UpdelRoll::mutex.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

                                                       {
   auto cd = chunk->getColumnDesc();
   auto td = catalog->getMetadataForTable(cd->tableId);
   auto data_buffer = chunk->getBuffer();
   std::lock_guard<std::mutex> lck(updel_roll.mutex);
   const auto key = std::make_pair(td, &fragment);
   if (0 == updel_roll.chunkMetadata.count(key)) {
     updel_roll.chunkMetadata[key] = fragment.getChunkMetadataMapPhysical();
   }
   auto& chunkMetadata = updel_roll.chunkMetadata[key];
   chunkMetadata[cd->columnId]->numElements = nrows_to_keep;
   chunkMetadata[cd->columnId]->numBytes = data_buffer->size();
   if (updel_roll.dirtyChunks.count(chunk.get()) == 0) {
     updel_roll.dirtyChunks.emplace(chunk.get(), chunk);
     ChunkKey chunk_key{
         catalog->getDatabaseId(), cd->tableId, cd->columnId, fragment.fragmentId};
     updel_roll.dirtyChunkeys.emplace(chunk_key);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

static void Fragmenter_Namespace::set_chunk_stats	(	const SQLTypeInfo &	col_type,
		int8_t *	data_addr,
		bool &	has_null,
		T &	min,
		T &	max
	)

static

Definition at line 1069 of file UpdelStorage.cpp.

References SQLTypeInfo::get_notnull(), is_null(), anonymous_namespace{TypedDataAccessors.h}::set_minmax(), and omnisci.dtypes::T.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

                                     {
   T v;
   const auto can_be_null = !col_type.get_notnull();
   const auto is_null = get_scalar<T>(data_addr, col_type, v);
   if (is_null) {
     has_null = has_null || (can_be_null && is_null);
   } else {
     set_minmax(min, max, v);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Fragmenter_Namespace::shuffleByIndexes	(	const ColumnDescriptor *	cd,
		const std::vector< size_t > &	indexes,
		DataBlockPtr &	data
	)

Definition at line 44 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, shuffleByIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

                                           {
   const auto& ti = cd->columnType;
   switch (ti.get_type()) {
     case kBOOLEAN:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kTINYINT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kSMALLINT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
       break;
     case kINT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
       break;
     case kBIGINT:
     case kNUMERIC:
     case kDECIMAL:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kFLOAT:
       shuffleByIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
       break;
     case kDOUBLE:
       shuffleByIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
       break;
     case kTEXT:
     case kVARCHAR:
     case kCHAR:
       if (ti.is_varlen()) {
         shuffleByIndexesImpl(indexes, *data.stringsPtr);
       } else {
         switch (ti.get_size()) {
           case 1:
             shuffleByIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
             break;
           case 2:
             shuffleByIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
             break;
           case 4:
             shuffleByIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
             break;
           default:
             CHECK(false);
         }
       }
       break;
     case kDATE:
     case kTIME:
     case kTIMESTAMP:
       shuffleByIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kARRAY:
       shuffleByIndexesImpl(indexes, *data.arraysPtr);
       break;
     case kPOINT:
     case kLINESTRING:
     case kPOLYGON:
     case kMULTIPOLYGON:
       shuffleByIndexesImpl(indexes, *data.stringsPtr);
       break;
     default:
       CHECK(false);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

void Fragmenter_Namespace::shuffleByIndexesImpl	(	const std::vector< size_t > &	indexes,
		T *	buffer
	)

Definition at line 25 of file SortedOrderFragmenter.cpp.

References i, and omnisci.dtypes::T.

Referenced by shuffleByIndexes().

                                                                        {
   std::vector<T> new_buffer;
   new_buffer.reserve(indexes.size());
   for (const auto i : indexes) {
     new_buffer.push_back(buffer[i]);
   }
   std::memcpy(buffer, new_buffer.data(), indexes.size() * sizeof(T));
 }

Here is the caller graph for this function:

template<typename T >

void Fragmenter_Namespace::shuffleByIndexesImpl	(	const std::vector< size_t > &	indexes,
		std::vector< T > &	buffer
	)

Definition at line 35 of file SortedOrderFragmenter.cpp.

References i.

                                                                                   {
   std::vector<T> new_buffer;
   new_buffer.reserve(indexes.size());
   for (const auto i : indexes) {
     new_buffer.push_back(buffer[i]);
   }
   buffer.swap(new_buffer);
 }

size_t Fragmenter_Namespace::sizeOfRawColumn	(	const Catalog_Namespace::Catalog &	cat,
		const ColumnDescriptor *	cd
	)

Definition at line 81 of file InsertDataLoader.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_compression(), SQLTypeInfo::get_logical_size(), SQLTypeInfo::get_size(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_NONE, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

Referenced by computeRowIndicesOfShards(), and copyColumnDataOfShard().

                                                    {
   switch (cd->columnType.get_type()) {
     case kPOINT:
     case kLINESTRING:
     case kPOLYGON:
     case kMULTIPOLYGON:
     case kARRAY:
       throw std::runtime_error("geo and array columns have variable length elements");
     case kBOOLEAN:
     case kTINYINT:
     case kSMALLINT:
     case kINT:
     case kBIGINT:
     case kNUMERIC:
     case kDECIMAL:
     case kFLOAT:
     case kDOUBLE:
     case kTIMESTAMP:
     case kTIME:
     case kINTERVAL_DAY_TIME:
     case kINTERVAL_YEAR_MONTH:
     case kDATE:
       return cd->columnType.get_logical_size();
     case kTEXT:
     case kVARCHAR:
     case kCHAR:
       if (cd->columnType.get_compression() == kENCODING_NONE) {
         throw std::runtime_error(
             "non encoded string columns have variable length elements");
       }
       return cd->columnType.get_size();
     default:
       throw std::runtime_error("not supported column type: " + cd->columnName + " (" +
                                cd->columnType.get_type_name() + ")");
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexes	(	const ColumnDescriptor *	cd,
		std::vector< size_t > &	indexes,
		const DataBlockPtr &	data
	)

Definition at line 137 of file SortedOrderFragmenter.cpp.

References DataBlockPtr::arraysPtr, CHECK, ColumnDescriptor::columnType, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, DataBlockPtr::numbersPtr, sortIndexesImpl(), and DataBlockPtr::stringsPtr.

Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().

                                            {
   const auto& ti = cd->columnType;
   switch (ti.get_type()) {
     case kBOOLEAN:
       sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kTINYINT:
       sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
       break;
     case kSMALLINT:
       sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
       break;
     case kINT:
       sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
       break;
     case kBIGINT:
     case kNUMERIC:
     case kDECIMAL:
       sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kFLOAT:
       sortIndexesImpl(indexes, reinterpret_cast<float*>(data.numbersPtr));
       break;
     case kDOUBLE:
       sortIndexesImpl(indexes, reinterpret_cast<double*>(data.numbersPtr));
       break;
     case kTEXT:
     case kVARCHAR:
     case kCHAR:
       if (ti.is_varlen()) {
         sortIndexesImpl(indexes, *data.stringsPtr);
       } else {
         switch (ti.get_size()) {
           case 1:
             sortIndexesImpl(indexes, reinterpret_cast<int8_t*>(data.numbersPtr));
             break;
           case 2:
             sortIndexesImpl(indexes, reinterpret_cast<int16_t*>(data.numbersPtr));
             break;
           case 4:
             sortIndexesImpl(indexes, reinterpret_cast<int32_t*>(data.numbersPtr));
             break;
           default:
             CHECK(false);
         }
       }
       break;
     case kDATE:
     case kTIME:
     case kTIMESTAMP:
       sortIndexesImpl(indexes, reinterpret_cast<int64_t*>(data.numbersPtr));
       break;
     case kARRAY:
       sortIndexesImpl(indexes, *data.arraysPtr);
       break;
     default:
       CHECK(false) << "invalid type '" << ti.get_type() << "' to sort";
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const T *	buffer
	)

Definition at line 113 of file SortedOrderFragmenter.cpp.

References CHECK, and gpu_enabled::sort().

Referenced by sortIndexes().

                                                                   {
   CHECK(buffer);
   std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
     return buffer[a] < buffer[b];
   });
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const std::vector< std::string > &	buffer
	)

Definition at line 120 of file SortedOrderFragmenter.cpp.

References gpu_enabled::sort().

                                                          {
   std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
     return buffer[a].size() < buffer[b].size() ||
            (buffer[a].size() == buffer[b].size() && buffer[a] < buffer[b]);
   });
 }

Here is the call graph for this function:

void Fragmenter_Namespace::sortIndexesImpl	(	std::vector< size_t > &	indexes,
		const std::vector< ArrayDatum > &	buffer
	)

Definition at line 128 of file SortedOrderFragmenter.cpp.

References gpu_enabled::sort().

                                                           {
   std::sort(indexes.begin(), indexes.end(), [&](const auto a, const auto b) {
     return buffer[a].is_null || buffer[a].length < buffer[b].length ||
            (!buffer[b].is_null && buffer[a].length == buffer[b].length &&
             memcmp(buffer[a].pointer, buffer[b].pointer, buffer[a].length) < 0);
   });
 }

Here is the call graph for this function:

void Fragmenter_Namespace::wait_cleanup_threads ( std::vector< std::future< void >> & threads )

inline

Definition at line 41 of file UpdelStorage.cpp.

References t.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows(), Fragmenter_Namespace::InsertOrderFragmenter::getVacuumOffsets(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

                                                                       {
   for (auto& t : threads) {
     t.get();
   }
   threads.clear();
 }

Here is the caller graph for this function:

Namespaces

Classes

Enumerations

Functions

Enumeration Type Documentation

Function Documentation