OmniSciDB
85c2d10cdc
|
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter. More...
#include <InsertOrderFragmenter.h>
Public Types | |
using | ModifyTransactionTracker = UpdelRoll |
Public Member Functions | |
InsertOrderFragmenter (const std::vector< int > chunkKeyPrefix, std::vector< Chunk_NS::Chunk > &chunkVec, Data_Namespace::DataMgr *dataMgr, Catalog_Namespace::Catalog *catalog, const int physicalTableId, const int shard, const size_t maxFragmentRows=DEFAULT_FRAGMENT_ROWS, const size_t maxChunkSize=DEFAULT_MAX_CHUNK_SIZE, const size_t pageSize=DEFAULT_PAGE_SIZE, const size_t maxRows=DEFAULT_MAX_ROWS, const Data_Namespace::MemoryLevel defaultInsertLevel=Data_Namespace::DISK_LEVEL, const bool uses_foreign_storage=false) | |
~InsertOrderFragmenter () override | |
TableInfo | getFragmentsForQuery () override |
returns (inside QueryInfo) object all ids and row sizes of fragments More... | |
void | insertData (InsertData &insert_data_struct) override |
appends data onto the most recently occuring fragment, creating a new one if necessary More... | |
void | insertDataNoCheckpoint (InsertData &insert_data_struct) override |
Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally. More... | |
void | dropFragmentsToSize (const size_t maxRows) override |
Will truncate table to less than maxRows by dropping fragments. More... | |
void | updateColumnChunkMetadata (const ColumnDescriptor *cd, const int fragment_id, const std::shared_ptr< ChunkMetadata > metadata) override |
Updates the metadata for a column chunk. More... | |
void | updateChunkStats (const ColumnDescriptor *cd, std::unordered_map< int, ChunkStats > &stats_map, std::optional< Data_Namespace::MemoryLevel > memory_level) override |
Update chunk stats. More... | |
FragmentInfo * | getFragmentInfo (const int fragment_id) const override |
Retrieve the fragment info object for an individual fragment for editing. More... | |
int | getFragmenterId () override |
get fragmenter's id More... | |
std::vector< int > | getChunkKeyPrefix () const |
std::string | getFragmenterType () override |
get fragmenter's type (as string More... | |
size_t | getNumRows () override |
void | setNumRows (const size_t numTuples) override |
std::optional< ChunkUpdateStats > | updateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const std::vector< ScalarTargetValue > &rhs_values, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override |
void | updateColumns (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< TargetMetaInfo > sourceMetaInfo, const std::vector< const ColumnDescriptor * > columnDescriptors, const RowDataProvider &sourceDataProvider, const size_t indexOffFragmentOffsetColumn, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll, Executor *executor) override |
void | updateColumn (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const ColumnDescriptor *cd, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const ScalarTargetValue &rhs_value, const SQLTypeInfo &rhs_type, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override |
void | updateColumnMetadata (const ColumnDescriptor *cd, FragmentInfo &fragment, std::shared_ptr< Chunk_NS::Chunk > chunk, const UpdateValuesStats &update_values_stats, const SQLTypeInfo &rhs_type, UpdelRoll &updel_roll) override |
void | updateMetadata (const Catalog_Namespace::Catalog *catalog, const MetaDataKey &key, UpdelRoll &updel_roll) override |
void | compactRows (const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragment_id, const std::vector< uint64_t > &frag_offsets, const Data_Namespace::MemoryLevel memory_level, UpdelRoll &updel_roll) override |
const std::vector< uint64_t > | getVacuumOffsets (const std::shared_ptr< Chunk_NS::Chunk > &chunk) override |
auto | getChunksForAllColumns (const TableDescriptor *td, const FragmentInfo &fragment, const Data_Namespace::MemoryLevel memory_level) |
void | dropColumns (const std::vector< int > &columnIds) override |
bool | hasDeletedRows (const int delete_column_id) override |
Iterates through chunk metadata to return whether any rows have been deleted. More... | |
![]() | |
virtual | ~AbstractFragmenter () |
Protected Attributes | |
std::vector< int > | chunkKeyPrefix_ |
std::map< int, Chunk_NS::Chunk > | columnMap_ |
std::deque< std::unique_ptr < FragmentInfo > > | fragmentInfoVec_ |
Data_Namespace::DataMgr * | dataMgr_ |
Catalog_Namespace::Catalog * | catalog_ |
const int | physicalTableId_ |
const int | shard_ |
size_t | maxFragmentRows_ |
size_t | pageSize_ |
size_t | numTuples_ |
int | maxFragmentId_ |
size_t | maxChunkSize_ |
size_t | maxRows_ |
std::string | fragmenterType_ |
mapd_shared_mutex | fragmentInfoMutex_ |
mapd_shared_mutex | insertMutex_ |
Data_Namespace::MemoryLevel | defaultInsertLevel_ |
const bool | uses_foreign_storage_ |
bool | hasMaterializedRowId_ |
int | rowIdColId_ |
std::unordered_map< int, size_t > | varLenColInfo_ |
std::shared_ptr< std::mutex > | mutex_access_inmem_states |
std::mutex | temp_mutex_ |
Private Member Functions | |
bool | isAddingNewColumns (const InsertData &insert_data) const |
void | dropFragmentsToSizeNoInsertLock (const size_t max_rows) |
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order. Likely the default fragmenter.
Definition at line 53 of file InsertOrderFragmenter.h.
Definition at line 55 of file InsertOrderFragmenter.h.
Fragmenter_Namespace::InsertOrderFragmenter::InsertOrderFragmenter | ( | const std::vector< int > | chunkKeyPrefix, |
std::vector< Chunk_NS::Chunk > & | chunkVec, | ||
Data_Namespace::DataMgr * | dataMgr, | ||
Catalog_Namespace::Catalog * | catalog, | ||
const int | physicalTableId, | ||
const int | shard, | ||
const size_t | maxFragmentRows = DEFAULT_FRAGMENT_ROWS , |
||
const size_t | maxChunkSize = DEFAULT_MAX_CHUNK_SIZE , |
||
const size_t | pageSize = DEFAULT_PAGE_SIZE , |
||
const size_t | maxRows = DEFAULT_MAX_ROWS , |
||
const Data_Namespace::MemoryLevel | defaultInsertLevel = Data_Namespace::DISK_LEVEL , |
||
const bool | uses_foreign_storage = false |
||
) |
|
override |
Definition at line 95 of file InsertOrderFragmenter.cpp.
|
protected |
|
protected |
Definition at line 453 of file InsertOrderFragmenter.cpp.
References catalog_(), CHECK, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, i, and Fragmenter_Namespace::InsertData::numRows.
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 1305 of file UpdelStorage.cpp.
References CHECK, cpu_threads(), anonymous_namespace{ResultSetReductionInterpreter.cpp}::get_element_size(), DateConverters::get_epoch_seconds_from_days(), getChunksForAllColumns(), getFragmentInfo(), UpdelRoll::numTuples, Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::set_chunk_stats(), updateColumnMetadata(), vacuum_fixlen_rows(), vacuum_varlen_rows(), and Fragmenter_Namespace::wait_cleanup_threads().
Referenced by updateColumn().
|
protected |
Definition at line 115 of file InsertOrderFragmenter.cpp.
References catalog_(), Data_Namespace::DISK_LEVEL, File_Namespace::FileMgrParams::max_rollback_epochs, and TableDescriptor::maxRollbackEpochs.
|
protected |
creates new fragment, calling createChunk() method of BufferMgr to make a new chunk for each column of the table.
Also unpins the chunks of the previous insert buffer
Definition at line 726 of file InsertOrderFragmenter.cpp.
References Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::compute_device_for_fragment(), and Fragmenter_Namespace::FragmentInfo::fragmentId.
|
protected |
Definition at line 252 of file InsertOrderFragmenter.cpp.
References catalog_(), and lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable().
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 540 of file InsertOrderFragmenter.cpp.
|
overridevirtual |
Will truncate table to less than maxRows by dropping fragments.
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 220 of file InsertOrderFragmenter.cpp.
|
private |
Definition at line 225 of file InsertOrderFragmenter.cpp.
References CHECK_GE, CHECK_GT, DROP_FRAGMENT_FACTOR, logger::INFO, and LOG.
|
inline |
Definition at line 108 of file InsertOrderFragmenter.h.
References chunkKeyPrefix_.
|
protected |
Definition at line 129 of file InsertOrderFragmenter.cpp.
References CHECK, CHECK_GE, Fragmenter_Namespace::anonymous_namespace{InsertOrderFragmenter.cpp}::compute_device_for_fragment(), Data_Namespace::DISK_LEVEL, logger::FATAL, LOG, gpu_enabled::sort(), and to_string().
auto Fragmenter_Namespace::InsertOrderFragmenter::getChunksForAllColumns | ( | const TableDescriptor * | td, |
const FragmentInfo & | fragment, | ||
const Data_Namespace::MemoryLevel | memory_level | ||
) |
Definition at line 1007 of file UpdelStorage.cpp.
References catalog_, CHECK, Catalog_Namespace::DBMetadata::dbId, Fragmenter_Namespace::FragmentInfo::fragmentId, Chunk_NS::Chunk::getChunk(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), TableDescriptor::nColumns, and TableDescriptor::tableId.
Referenced by compactRows().
|
inlineoverridevirtual |
get fragmenter's id
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 107 of file InsertOrderFragmenter.h.
References chunkKeyPrefix_.
|
inlineoverridevirtual |
get fragmenter's type (as string
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 112 of file InsertOrderFragmenter.h.
References fragmenterType_.
|
overridevirtual |
Retrieve the fragment info object for an individual fragment for editing.
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 389 of file InsertOrderFragmenter.cpp.
References CHECK.
Referenced by compactRows(), updateColumn(), and updateColumns().
|
protected |
|
overridevirtual |
returns (inside QueryInfo) object all ids and row sizes of fragments
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 762 of file InsertOrderFragmenter.cpp.
References Fragmenter_Namespace::TableInfo::chunkKeyPrefix, Fragmenter_Namespace::FragmentInfo::deviceIds, Fragmenter_Namespace::FragmentInfo::fragmentId, Fragmenter_Namespace::TableInfo::fragments, Fragmenter_Namespace::TableInfo::getPhysicalNumTuples(), Fragmenter_Namespace::FragmentInfo::physicalTableId, Fragmenter_Namespace::FragmentInfo::setPhysicalNumTuples(), Fragmenter_Namespace::TableInfo::setPhysicalNumTuples(), Fragmenter_Namespace::FragmentInfo::shadowNumTuples, and Fragmenter_Namespace::FragmentInfo::shard.
|
inlineoverridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 113 of file InsertOrderFragmenter.h.
References numTuples_.
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 1036 of file UpdelStorage.cpp.
References CHECK, cpu_threads(), i, test_fsi::r, and Fragmenter_Namespace::wait_cleanup_threads().
Referenced by updateColumn().
|
overridevirtual |
Iterates through chunk metadata to return whether any rows have been deleted.
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 571 of file InsertOrderFragmenter.cpp.
References CHECK.
|
overridevirtual |
appends data onto the most recently occuring fragment, creating a new one if necessary
Implements Fragmenter_Namespace::AbstractFragmenter.
Reimplemented in Fragmenter_Namespace::SortedOrderFragmenter.
Definition at line 414 of file InsertOrderFragmenter.cpp.
References catalog_(), Fragmenter_Namespace::InsertData::databaseId, Data_Namespace::DISK_LEVEL, and Fragmenter_Namespace::InsertData::tableId.
Referenced by Fragmenter_Namespace::SortedOrderFragmenter::insertData().
|
protected |
Definition at line 585 of file InsertOrderFragmenter.cpp.
References CHECK, CHECK_GT, CHECK_LE, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, i, Fragmenter_Namespace::InsertData::is_default, DataBlockPtr::numbersPtr, and Fragmenter_Namespace::InsertData::numRows.
|
overridevirtual |
Given data wrapped in an InsertData struct, inserts it into the correct partitions No locks and checkpoints taken needs to be managed externally.
Implements Fragmenter_Namespace::AbstractFragmenter.
Reimplemented in Fragmenter_Namespace::SortedOrderFragmenter.
Definition at line 441 of file InsertOrderFragmenter.cpp.
Referenced by Fragmenter_Namespace::SortedOrderFragmenter::insertDataNoCheckpoint(), and updateColumns().
|
private |
Definition at line 399 of file InsertOrderFragmenter.cpp.
References CHECK, and Fragmenter_Namespace::InsertData::columnIds.
|
protected |
|
protected |
|
inlineoverridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 114 of file InsertOrderFragmenter.h.
References numTuples_.
|
overridevirtual |
Update chunk stats.
WARNING: This method is entirely unlocked. Higher level locks are expected to prevent any table read or write during a chunk metadata update, since we need to modify various buffers and metadata maps.
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 298 of file InsertOrderFragmenter.cpp.
References catalog_(), CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnType, DatumToString(), Data_Namespace::DISK_LEVEL, get_logical_type_info(), Chunk_NS::Chunk::getChunk(), SQLTypeInfo::is_dict_encoded_string(), kBIGINT, LOG, show_chunk(), VLOG, and logger::WARNING.
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 638 of file UpdelStorage.cpp.
References UpdelRoll::catalog, CHECK, CHECK_GT, Fragmenter_Namespace::ChunkUpdateStats::chunk, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, compactRows(), Data_Namespace::CPU_LEVEL, cpu_threads(), Catalog_Namespace::DBMetadata::dbId, anonymous_namespace{TypedDataAccessors.h}::decimal_to_double(), UpdelRoll::dirtyChunkeys, UpdelRoll::dirtyChunks, Fragmenter_Namespace::ChunkUpdateStats::fragment_rows_count, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), anonymous_namespace{ResultSetReductionInterpreter.cpp}::get_element_size(), DateConverters::get_epoch_seconds_from_days(), SQLTypeInfo::get_scale(), Chunk_NS::Chunk::getChunk(), Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), getFragmentInfo(), Catalog_Namespace::Catalog::getLogicalTableId(), Catalog_Namespace::Catalog::getMetadataForColumn(), Catalog_Namespace::Catalog::getMetadataForDict(), getVacuumOffsets(), SQLTypeInfo::is_boolean(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_fp(), Fragmenter_Namespace::is_integral(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), ColumnDescriptor::isDeletedCol, kENCODING_DICT, UpdelRoll::logicalTableId, UpdelRoll::memoryLevel, UpdelRoll::mutex, Fragmenter_Namespace::ChunkUpdateStats::new_values_stats, Fragmenter_Namespace::ChunkUpdateStats::old_values_stats, anonymous_namespace{TypedDataAccessors.h}::put_null(), test_fsi::r, shard_, ColumnDescriptor::tableId, TableDescriptor::tableId, temp_mutex_, to_string(), Fragmenter_Namespace::FragmentInfo::unconditionalVacuum_, UNREACHABLE, Fragmenter_Namespace::anonymous_namespace{UpdelStorage.cpp}::update_metadata(), foreign_storage::update_stats(), updateColumnMetadata(), Fragmenter_Namespace::ChunkUpdateStats::updated_rows_count, NullAwareValidator< INNER_VALIDATOR >::validate(), and Fragmenter_Namespace::wait_cleanup_threads().
Referenced by updateColumn().
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 54 of file UpdelStorage.cpp.
References updateColumn().
|
overridevirtual |
Updates the metadata for a column chunk.
cd | - ColumnDescriptor for the column |
fragment_id | - Fragment id of the chunk within the column |
metadata | - shared_ptr of the metadata to update column chunk with |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 285 of file InsertOrderFragmenter.cpp.
References CHECK, and ColumnDescriptor::columnId.
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 937 of file UpdelStorage.cpp.
References UpdelRoll::catalog, UpdelRoll::chunkMetadata, ColumnDescriptor::columnId, ColumnDescriptor::columnType, Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysical(), Catalog_Namespace::Catalog::getMetadataForTable(), Fragmenter_Namespace::UpdateValuesStats::has_null, SQLTypeInfo::is_decimal(), Fragmenter_Namespace::is_integral(), kENCODING_DICT, Fragmenter_Namespace::UpdateValuesStats::max_double, Fragmenter_Namespace::UpdateValuesStats::max_int64t, Fragmenter_Namespace::UpdateValuesStats::min_double, Fragmenter_Namespace::UpdateValuesStats::min_int64t, UpdelRoll::mutex, UpdelRoll::numTuples, Fragmenter_Namespace::FragmentInfo::shadowNumTuples, ColumnDescriptor::tableId, and foreign_storage::update_stats().
Referenced by compactRows(), and updateColumn().
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 267 of file UpdelStorage.cpp.
References UpdelRoll::catalog, CHECK, checked_get(), Fragmenter_Namespace::InsertData::columnIds, cpu_threads(), TargetValueConverterFactory::create(), Fragmenter_Namespace::InsertData::databaseId, Catalog_Namespace::DBMetadata::dbId, UpdelRoll::dirtyChunkeys, UpdelRoll::dirtyChunks, g_enable_experimental_string_functions, Fragmenter_Namespace::get_chunks(), get_logical_type_info(), SQLTypeInfo::get_size(), Catalog_Namespace::Catalog::getCurrentDB(), Fragmenter_Namespace::RowDataProvider::getEntryAt(), Fragmenter_Namespace::RowDataProvider::getEntryCount(), getFragmentInfo(), Fragmenter_Namespace::RowDataProvider::getLiteralDictionary(), Catalog_Namespace::Catalog::getLogicalTableId(), Fragmenter_Namespace::RowDataProvider::getRowCount(), i, insertDataNoCheckpoint(), Fragmenter_Namespace::InsertData::is_default, SQLTypeInfo::is_string(), UpdelRoll::is_varlen_update, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, UpdelRoll::logicalTableId, UpdelRoll::memoryLevel, Fragmenter_Namespace::InsertData::numRows, TableDescriptor::tableId, and Fragmenter_Namespace::InsertData::tableId.
|
overridevirtual |
Implements Fragmenter_Namespace::AbstractFragmenter.
Definition at line 993 of file UpdelStorage.cpp.
References UpdelRoll::chunkMetadata, fragmentInfoMutex_, and UpdelRoll::numTuples.
|
protected |
Definition at line 1108 of file UpdelStorage.cpp.
References anonymous_namespace{ResultSetReductionInterpreter.cpp}::get_element_size(), and Fragmenter_Namespace::FragmentInfo::getPhysicalNumTuples().
Referenced by compactRows().
|
protected |
Definition at line 1222 of file UpdelStorage.cpp.
References CHECK, Fragmenter_Namespace::get_buffer_offset(), Fragmenter_Namespace::get_null_padding(), Fragmenter_Namespace::get_var_len_null_array_indexes(), Fragmenter_Namespace::FragmentInfo::getPhysicalNumTuples(), and i.
Referenced by compactRows().
|
protected |
Definition at line 185 of file InsertOrderFragmenter.h.
Referenced by getChunksForAllColumns(), and Fragmenter_Namespace::SortedOrderFragmenter::sortData().
|
protected |
Definition at line 178 of file InsertOrderFragmenter.h.
Referenced by getChunkKeyPrefix(), and getFragmenterId().
|
protected |
stores a map of column id to metadata about that column
Definition at line 180 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 184 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 201 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 195 of file InsertOrderFragmenter.h.
Referenced by getFragmenterType().
|
protected |
Definition at line 197 of file InsertOrderFragmenter.h.
Referenced by updateMetadata().
|
protected |
data about each fragment stored - id and number of rows
Definition at line 182 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 203 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 199 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 193 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 192 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 188 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 194 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 206 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 191 of file InsertOrderFragmenter.h.
Referenced by getNumRows(), and setNumRows().
|
protected |
Definition at line 189 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 186 of file InsertOrderFragmenter.h.
Referenced by Fragmenter_Namespace::SortedOrderFragmenter::sortData().
|
protected |
Definition at line 204 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 187 of file InsertOrderFragmenter.h.
Referenced by updateColumn().
|
mutableprotected |
Definition at line 230 of file InsertOrderFragmenter.h.
Referenced by updateColumn().
|
protected |
Definition at line 202 of file InsertOrderFragmenter.h.
|
protected |
Definition at line 205 of file InsertOrderFragmenter.h.