OmniSciDB  b24e664e58
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Fragmenter_Namespace::SortedOrderFragmenter Class Reference

#include <SortedOrderFragmenter.h>

+ Inheritance diagram for Fragmenter_Namespace::SortedOrderFragmenter:
+ Collaboration diagram for Fragmenter_Namespace::SortedOrderFragmenter:

Public Member Functions

 SortedOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr *dataMgr, Catalog_Namespace::Catalog *catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL)
 
 ~SortedOrderFragmenter () override
 
void insertData (InsertData &insertDataStruct) override
 appends data onto the most recently occuring fragment, creating a new one if necessary More...
 
void insertDataNoCheckpoint (InsertData &insertDataStruct) override
 Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally. More...
 
 SortedOrderFragmenter (SortedOrderFragmenter &&)=default
 
 SortedOrderFragmenter (const SortedOrderFragmenter &)=delete
 
SortedOrderFragmenteroperator= (const SortedOrderFragmenter &)=delete
 
- Public Member Functions inherited from Fragmenter_Namespace::InsertOrderFragmenter
 InsertOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr *dataMgr, Catalog_Namespace::Catalog *catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL)
 
 ~InsertOrderFragmenter () override
 
TableInfo getFragmentsForQuery () override
 returns (inside QueryInfo) object all ids and row sizes of fragments More...
 
void dropFragmentsToSize (const size_t maxRows) override
 Will truncate table to less than maxRows by dropping fragments. More...
 
void updateChunkStats (const ColumnDescriptor *cd, std::unordered_map< int, ChunkStats > &stats_map) override
 Update chunk stats. More...
 
int getFragmenterId () override
 get fragmenter's id More...
 
std::vector< int > getChunkKeyPrefix () const
 
std::string getFragmenterType () override
 get fragmenter's type (as string More...
 
size_t getNumRows () override
 
void setNumRows (const size_t numTuples) override
 
void updateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const std::vector< ScalarTargetValue > &rhs_values, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
 
void updateColumns (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< TargetMetaInfo > sourceMetaInfo, const std::vector< const ColumnDescriptor * > columnDescriptors, const RowDataProvider &sourceDataProvider, const size_t indexOffFragmentOffsetColumn, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll) override
 
void updateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const ScalarTargetValue &rhs_value, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
 
void updateColumnMetadata (const ColumnDescriptor *cd, FragmentInfo &fragment, std::shared_ptr< Chunk_NS::Chunk > chunk, const bool null, const double dmax, const double dmin, const int64_t lmax, const int64_t lmin, const SQLTypeInfo &rhs_type, UpdelRoll &updel_roll) override
 
void updateMetadata (const Catalog_Namespace::Catalog *catalog, const MetaDataKey &key, UpdelRoll &updel_roll) override
 
void compactRows (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override
 
const std::vector< uint64_t > getVacuumOffsets (const std::shared_ptr< Chunk_NS::Chunk > &chunk) override
 
auto getChunksForAllColumns (const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level)
 
- Public Member Functions inherited from Fragmenter_Namespace::AbstractFragmenter
virtual ~AbstractFragmenter ()
 

Protected Member Functions

virtual void sortData (InsertData &insertDataStruct)
 
- Protected Member Functions inherited from Fragmenter_Namespace::InsertOrderFragmenter
FragmentInfocreateNewFragment (const Data_Namespace::MemoryLevel memory_level=Data_Namespace::DISK_LEVEL)
 creates new fragment, calling createChunk() method of BufferMgr to make a new chunk for each column of the table. More...
 
void deleteFragments (const std::vector< int > &dropFragIds)
 
void getChunkMetadata ()
 
void lockInsertCheckpointData (const InsertData &insertDataStruct)
 
void insertDataImpl (InsertData &insertDataStruct)
 
void replicateData (const InsertData &insertDataStruct)
 
 InsertOrderFragmenter (const InsertOrderFragmenter &)
 
InsertOrderFragmenteroperator= (const InsertOrderFragmenter &)
 
FragmentInfogetFragmentInfoFromId (const int fragment_id)
 
auto vacuum_fixlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)
 
auto vacuum_varlen_rows (const FragmentInfo &fragment, const std::shared_ptr< Chunk_NS::Chunk > &chunk, const std::vector< uint64_t > &frag_offsets)
 

Additional Inherited Members

- Public Types inherited from Fragmenter_Namespace::InsertOrderFragmenter
using ModifyTransactionTracker = UpdelRoll
 
- Static Public Member Functions inherited from Fragmenter_Namespace::InsertOrderFragmenter
static void updateColumn (const Catalog_Namespace::Catalog *catalog, const std::string &tab_name, const std::string &col_name, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const std::vector< ScalarTargetValue > &rhs_values, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll)
 
- Protected Attributes inherited from Fragmenter_Namespace::InsertOrderFragmenter
std::vector< int > chunkKeyPrefix_
 
std::map< int, Chunk_NS::ChunkcolumnMap_
 
std::deque< FragmentInfofragmentInfoVec_
 
Data_Namespace::DataMgrdataMgr_
 
Catalog_Namespace::Catalogcatalog_
 
const int physicalTableId_
 
const int shard_
 
size_t maxFragmentRows_
 
size_t pageSize_
 
size_t numTuples_
 
int maxFragmentId_
 
size_t maxChunkSize_
 
size_t maxRows_
 
std::string fragmenterType_
 
mapd_shared_mutex fragmentInfoMutex_
 
mapd_shared_mutex insertMutex_
 
Data_Namespace::MemoryLevel defaultInsertLevel_
 
bool hasMaterializedRowId_
 
int rowIdColId_
 
std::unordered_map< int, size_t > varLenColInfo_
 
std::shared_ptr< std::mutex > mutex_access_inmem_states
 
std::mutex temp_mutex_
 

Detailed Description

Definition at line 22 of file SortedOrderFragmenter.h.

Constructor & Destructor Documentation

Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( const std::vector< int >  chunkKeyPrefix,
std::vector< Chunk_NS::Chunk > &  chunkVec,
Data_Namespace::DataMgr dataMgr,
Catalog_Namespace::Catalog catalog,
const int  physicalTableId,
const int  shard,
const size_t  maxFragmentRows = DEFAULT_FRAGMENT_ROWS,
const size_t  maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
const size_t  pageSize = DEFAULT_PAGE_SIZE,
const size_t  maxRows = DEFAULT_MAX_ROWS,
const Data_Namespace::MemoryLevel  defaultInsertLevel = Data_Namespace::DISK_LEVEL 
)
inline

Definition at line 24 of file SortedOrderFragmenter.h.

36  : InsertOrderFragmenter(chunkKeyPrefix,
37  chunkVec,
38  dataMgr,
39  catalog,
40  physicalTableId,
41  shard,
42  maxFragmentRows,
43  maxChunkSize,
44  pageSize,
45  maxRows,
46  defaultInsertLevel) {}
InsertOrderFragmenter(const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr *dataMgr, Catalog_Namespace::Catalog *catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL)
Fragmenter_Namespace::SortedOrderFragmenter::~SortedOrderFragmenter ( )
inlineoverride

Definition at line 48 of file SortedOrderFragmenter.h.

48 {}
Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( SortedOrderFragmenter &&  )
default
Fragmenter_Namespace::SortedOrderFragmenter::SortedOrderFragmenter ( const SortedOrderFragmenter )
delete

Member Function Documentation

void Fragmenter_Namespace::SortedOrderFragmenter::insertData ( InsertData insertDataStruct)
inlineoverridevirtual

appends data onto the most recently occuring fragment, creating a new one if necessary

Todo:
be able to fill up current fragment in multi-row insert before creating new fragment

Reimplemented from Fragmenter_Namespace::InsertOrderFragmenter.

Definition at line 49 of file SortedOrderFragmenter.h.

References Fragmenter_Namespace::InsertOrderFragmenter::insertData(), and sortData().

49  {
50  sortData(insertDataStruct);
51  InsertOrderFragmenter::insertData(insertDataStruct);
52  }
void insertData(InsertData &insertDataStruct) override
appends data onto the most recently occuring fragment, creating a new one if necessary ...
virtual void sortData(InsertData &insertDataStruct)

+ Here is the call graph for this function:

void Fragmenter_Namespace::SortedOrderFragmenter::insertDataNoCheckpoint ( InsertData insertDataStruct)
inlineoverridevirtual

Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally.

Reimplemented from Fragmenter_Namespace::InsertOrderFragmenter.

Definition at line 54 of file SortedOrderFragmenter.h.

References Fragmenter_Namespace::InsertOrderFragmenter::insertDataNoCheckpoint(), and sortData().

54  {
55  sortData(insertDataStruct);
57  }
void insertDataNoCheckpoint(InsertData &insertDataStruct) override
Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and check...
virtual void sortData(InsertData &insertDataStruct)

+ Here is the call graph for this function:

SortedOrderFragmenter& Fragmenter_Namespace::SortedOrderFragmenter::operator= ( const SortedOrderFragmenter )
delete
void Fragmenter_Namespace::SortedOrderFragmenter::sortData ( InsertData insertDataStruct)
protectedvirtual

Definition at line 198 of file SortedOrderFragmenter.cpp.

References Fragmenter_Namespace::InsertOrderFragmenter::catalog_, CHECK(), CHECK_GT, CHECK_LT, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, Catalog_Namespace::Catalog::getMetadataForColumn(), Catalog_Namespace::Catalog::getMetadataForTable(), Fragmenter_Namespace::InsertData::numRows, Fragmenter_Namespace::InsertOrderFragmenter::physicalTableId_, Fragmenter_Namespace::shuffleByIndexes(), and Fragmenter_Namespace::sortIndexes().

Referenced by insertData(), and insertDataNoCheckpoint().

198  {
199  // coming here table must have defined a sort_column for mini sort
200  const auto table_desc = catalog_->getMetadataForTable(physicalTableId_);
201  CHECK(table_desc);
202  CHECK_GT(table_desc->sortedColumnId, 0);
203  const auto logical_cd =
204  catalog_->getMetadataForColumn(table_desc->tableId, table_desc->sortedColumnId);
205  CHECK(logical_cd);
206  const auto physical_cd = catalog_->getMetadataForColumn(
207  table_desc->tableId,
208  table_desc->sortedColumnId + (logical_cd->columnType.is_geometry() ? 1 : 0));
209  const auto it = std::find(insertDataStruct.columnIds.begin(),
210  insertDataStruct.columnIds.end(),
211  physical_cd->columnId);
212  CHECK(it != insertDataStruct.columnIds.end());
213  // sort row indexes of the sort column
214  std::vector<size_t> indexes(insertDataStruct.numRows);
215  std::iota(indexes.begin(), indexes.end(), 0);
216  const auto dist = std::distance(insertDataStruct.columnIds.begin(), it);
217  CHECK_LT(static_cast<size_t>(dist), insertDataStruct.data.size());
218  sortIndexes(physical_cd, indexes, insertDataStruct.data[dist]);
219  // shuffle rows of all columns
220  for (size_t i = 0; i < insertDataStruct.columnIds.size(); ++i) {
221  const auto cd = catalog_->getMetadataForColumn(table_desc->tableId,
222  insertDataStruct.columnIds[i]);
223  shuffleByIndexes(cd, indexes, insertDataStruct.data[i]);
224  }
225 }
void shuffleByIndexes(const ColumnDescriptor *cd, const std::vector< size_t > &indexes, DataBlockPtr &data)
#define CHECK_GT(x, y)
Definition: Logger.h:202
CHECK(cgen_state)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void sortIndexes(const ColumnDescriptor *cd, std::vector< size_t > &indexes, const DataBlockPtr &data)
#define CHECK_LT(x, y)
Definition: Logger.h:200
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:


The documentation for this class was generated from the following files: