Fragmenter_Namespace::SortedOrderFragmenter Class Reference

#include <SortedOrderFragmenter.h>

Inheritance diagram for Fragmenter_Namespace::SortedOrderFragmenter:

Collaboration diagram for Fragmenter_Namespace::SortedOrderFragmenter:

Public Member Functions
	SortedOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr *dataMgr, Catalog_Namespace::Catalog *catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL)
	~SortedOrderFragmenter () override
void	insertData (InsertData &insert_data_struct) override
	appends data onto the most recently occuring fragment, creating a new one if necessary More...
void	insertDataNoCheckpoint (InsertData &insert_data_struct) override
	Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally. More...
	SortedOrderFragmenter (SortedOrderFragmenter &&)=default
	SortedOrderFragmenter (const SortedOrderFragmenter &)=delete
SortedOrderFragmenter &	operator= (const SortedOrderFragmenter &)=delete
Public Member Functions inherited from Fragmenter_Namespace::InsertOrderFragmenter
	InsertOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr *dataMgr, Catalog_Namespace::Catalog *catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL, const bool uses_foreign_storage=false)
	~InsertOrderFragmenter () override
TableInfo	getFragmentsForQuery () override
	returns (inside QueryInfo) object all ids and row sizes of fragments More...
void	dropFragmentsToSize (const size_t maxRows) override
	Will truncate table to less than maxRows by dropping fragments. More...
void	updateColumnChunkMetadata (const ColumnDescriptor *cd, const int fragment_id, const std::shared_ptr< ChunkMetadata > metadata) override
	Updates the metadata for a column chunk. More...
void	updateChunkStats (const ColumnDescriptor *cd, std::unordered_map< int, ChunkStats > &stats_map, std::optional< Data_Namespace::MemoryLevel > memory_level) override
	Update chunk stats. More...
FragmentInfo *	getFragmentInfo (const int fragment_id) const override
	Retrieve the fragment info object for an individual fragment for editing. More...
int	getFragmenterId () override
	get fragmenter's id More...
std::vector< int >	getChunkKeyPrefix () const
std::string	getFragmenterType () override
	get fragmenter's type (as string More...
size_t	getNumRows () override
void	setNumRows (const size_t numTuples) override
std::optional< ChunkUpdateStats >	updateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const std::vector< ScalarTargetValue > &rhs_values, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
void	updateColumns (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< TargetMetaInfo > sourceMetaInfo, const std::vector< const ColumnDescriptor * > columnDescriptors, const RowDataProvider &sourceDataProvider, const size_t indexOffFragmentOffsetColumn, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll, Executor *executor) override
void	updateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const ScalarTargetValue &rhs_value, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
void	updateColumnMetadata (const ColumnDescriptor *cd, FragmentInfo &fragment, std::shared_ptr< Chunk_NS::Chunk > chunk, const UpdateValuesStats &update_values_stats, const SQLTypeInfo &rhs_type, UpdelRoll &updel_roll) override
void	updateMetadata (const Catalog_Namespace::Catalog *catalog, const MetaDataKey &key, UpdelRoll &updel_roll) override
void	compactRows (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
const std::vector< uint64_t >	getVacuumOffsets (const std::shared_ptr< Chunk_NS::Chunk > &chunk) override
auto	getChunksForAllColumns (const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level)
void	dropColumns (const std::vector< int > &columnIds) override
bool	hasDeletedRows (const int delete_column_id) override
	Iterates through chunk metadata to return whether any rows have been deleted. More...
Public Member Functions inherited from Fragmenter_Namespace::AbstractFragmenter
virtual	~AbstractFragmenter ()

Protected Member Functions
virtual void	sortData (InsertData &insertDataStruct)
Protected Member Functions inherited from Fragmenter_Namespace::InsertOrderFragmenter
FragmentInfo *	createNewFragment (const Data_Namespace::MemoryLevel memory_level=Data_Namespace::DISK_LEVEL)
	creates new fragment, calling createChunk() method of BufferMgr to make a new chunk for each column of the table. More...
void	deleteFragments (const std::vector< int > &dropFragIds)
void	conditionallyInstantiateFileMgrWithParams ()
void	getChunkMetadata ()
void	lockInsertCheckpointData (const InsertData &insertDataStruct)
void	insertDataImpl (InsertData &insert_data)
void	addColumns (const InsertData &insertDataStruct)
	InsertOrderFragmenter (const InsertOrderFragmenter &)
InsertOrderFragmenter &	operator= (const InsertOrderFragmenter &)
FragmentInfo &	getFragmentInfoFromId (const int fragment_id)
auto	vacuum_fixlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)
auto	vacuum_varlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)

Additional Inherited Members
Public Types inherited from Fragmenter_Namespace::InsertOrderFragmenter
using	ModifyTransactionTracker = UpdelRoll
Protected Attributes inherited from Fragmenter_Namespace::InsertOrderFragmenter
std::vector< int >	chunkKeyPrefix_
std::map< int, Chunk_NS::Chunk >	columnMap_
std::deque< std::unique_ptr < FragmentInfo > >	fragmentInfoVec_
Data_Namespace::DataMgr *	dataMgr_
Catalog_Namespace::Catalog *	catalog_
const int	physicalTableId_
const int	shard_
size_t	maxFragmentRows_
size_t	pageSize_
size_t	numTuples_
int	maxFragmentId_
size_t	maxChunkSize_
size_t	maxRows_
std::string	fragmenterType_
mapd_shared_mutex	fragmentInfoMutex_
mapd_shared_mutex	insertMutex_
Data_Namespace::MemoryLevel	defaultInsertLevel_
const bool	uses_foreign_storage_
bool	hasMaterializedRowId_
int	rowIdColId_
std::unordered_map< int, size_t >	varLenColInfo_
std::shared_ptr< std::mutex >	mutex_access_inmem_states
std::mutex	temp_mutex_

Detailed Description

Definition at line 22 of file SortedOrderFragmenter.h.

Constructor & Destructor Documentation

Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( const std::vector< int >  chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &  chunkVec, Data_Namespace::DataMgr *  dataMgr, Catalog_Namespace::Catalog *  catalog, const int  physicalTableId, const int  shard, const size_t  maxFragmentRows = DEFAULT_FRAGMENT_ROWS, const size_t  maxChunkSize = DEFAULT_MAX_CHUNK_SIZE, const size_t  pageSize = DEFAULT_PAGE_SIZE, const size_t  maxRows = DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel  defaultInsertLevel = Data_Namespace::DISK_LEVEL  )

inline

Definition at line 24 of file SortedOrderFragmenter.h.

      : InsertOrderFragmenter(chunkKeyPrefix,
                              chunkVec,
                              dataMgr,
                              catalog,
                              physicalTableId,
                              shard,
                              maxFragmentRows,
                              maxChunkSize,
                              pageSize,
                              maxRows,
                              defaultInsertLevel) {}

Fragmenter_Namespace::SortedOrderFragmenter::~SortedOrderFragmenter ( )

inlineoverride

Definition at line 48 of file SortedOrderFragmenter.h.

{}

Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( SortedOrderFragmenter &&  )

default

Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( const SortedOrderFragmenter &  )

delete

Member Function Documentation

void Fragmenter_Namespace::SortedOrderFragmenter::insertData ( InsertData &  insert_data_struct )

inlineoverridevirtual

appends data onto the most recently occuring fragment, creating a new one if necessary

Todo:

be able to fill up current fragment in multi-row insert before creating new fragment

Reimplemented from Fragmenter_Namespace::InsertOrderFragmenter.

Definition at line 49 of file SortedOrderFragmenter.h.

References Fragmenter_Namespace::InsertOrderFragmenter::insertData(), and sortData().

                                                           {
    sortData(insert_data_struct);
    InsertOrderFragmenter::insertData(insert_data_struct);
  }

Here is the call graph for this function:

void Fragmenter_Namespace::SortedOrderFragmenter::insertDataNoCheckpoint ( InsertData &  insert_data_struct )

inlineoverridevirtual

Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally.

Reimplemented from Fragmenter_Namespace::InsertOrderFragmenter.

Definition at line 54 of file SortedOrderFragmenter.h.

References Fragmenter_Namespace::InsertOrderFragmenter::insertDataNoCheckpoint(), and sortData().

                                                                       {
    sortData(insert_data_struct);
    InsertOrderFragmenter::insertDataNoCheckpoint(insert_data_struct);
  }

Here is the call graph for this function:

SortedOrderFragmenter& Fragmenter_Namespace::SortedOrderFragmenter::operator= ( const SortedOrderFragmenter &  )

delete

void Fragmenter_Namespace::SortedOrderFragmenter::sortData ( InsertData &  insertDataStruct )

protectedvirtual

Definition at line 199 of file SortedOrderFragmenter.cpp.

References Fragmenter_Namespace::InsertOrderFragmenter::catalog_, CHECK, CHECK_GT, CHECK_LT, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, Catalog_Namespace::Catalog::getMetadataForColumn(), Catalog_Namespace::Catalog::getMetadataForTable(), i, gpu_enabled::iota(), Fragmenter_Namespace::InsertData::is_default, Fragmenter_Namespace::InsertData::numRows, Fragmenter_Namespace::InsertOrderFragmenter::physicalTableId_, Fragmenter_Namespace::shuffleByIndexes(), and Fragmenter_Namespace::sortIndexes().

Referenced by insertData(), and insertDataNoCheckpoint().

                                                                 {
  // coming here table must have defined a sort_column for mini sort
  const auto table_desc = catalog_->getMetadataForTable(physicalTableId_);
  CHECK(table_desc);
  CHECK_GT(table_desc->sortedColumnId, 0);
  const auto logical_cd =
      catalog_->getMetadataForColumn(table_desc->tableId, table_desc->sortedColumnId);
  CHECK(logical_cd);
  const auto physical_cd = catalog_->getMetadataForColumn(
      table_desc->tableId,
      table_desc->sortedColumnId + (logical_cd->columnType.is_geometry() ? 1 : 0));
  const auto it = std::find(insertDataStruct.columnIds.begin(),
                            insertDataStruct.columnIds.end(),
                            physical_cd->columnId);
  CHECK(it != insertDataStruct.columnIds.end());
  // sort row indexes of the sort column
  const auto dist = std::distance(insertDataStruct.columnIds.begin(), it);
  if (!insertDataStruct.is_default[dist]) {
    std::vector<size_t> indexes(insertDataStruct.numRows);
    std::iota(indexes.begin(), indexes.end(), 0);
    CHECK_LT(static_cast<size_t>(dist), insertDataStruct.data.size());
    sortIndexes(physical_cd, indexes, insertDataStruct.data[dist]);
    // shuffle rows of all columns
    for (size_t i = 0; i < insertDataStruct.columnIds.size(); ++i) {
      if (insertDataStruct.is_default[i]) {
        continue;
      }
      const auto cd = catalog_->getMetadataForColumn(table_desc->tableId,
                                                     insertDataStruct.columnIds[i]);
      shuffleByIndexes(cd, indexes, insertDataStruct.data[i]);
    }
  } else {
    // nothing to shuffle, the column has the same value across all rows
    return;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/Fragmenter/SortedOrderFragmenter.h
• /home/jenkins-slave/workspace/core-os-doxygen/Fragmenter/SortedOrderFragmenter.cpp