OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
UpdelRoll Struct Reference

#include <UpdelRoll.h>

+ Collaboration diagram for UpdelRoll:

Public Member Functions

 ~UpdelRoll ()
 
void cancelUpdate ()
 
bool commitUpdate ()
 
void stageUpdate ()
 
void addDirtyChunk (std::shared_ptr< Chunk_NS::Chunk > chunk, int fragment_id)
 
std::shared_ptr< ChunkMetadatagetChunkMetadata (const MetaDataKey &key, int32_t column_id, Fragmenter_Namespace::FragmentInfo &fragment_info)
 
ChunkMetadataMap getChunkMetadataMap (const MetaDataKey &key) const
 
size_t getNumTuple (const MetaDataKey &key) const
 
void setNumTuple (const MetaDataKey &key, size_t num_tuple)
 

Public Attributes

const Catalog_Namespace::Catalogcatalog = nullptr
 
int logicalTableId
 
Data_Namespace::MemoryLevel memoryLevel {Data_Namespace::MemoryLevel::CPU_LEVEL}
 
bool is_varlen_update = false
 
const TableDescriptortable_descriptor {nullptr}
 

Private Member Functions

void updateFragmenterAndCleanupChunks ()
 
void initializeUnsetMetadata (const TableDescriptor *td, Fragmenter_Namespace::FragmentInfo &fragment_info)
 

Private Attributes

heavyai::shared_mutex chunk_update_tracker_mutex
 
std::map< ChunkKey,
std::shared_ptr
< Chunk_NS::Chunk > > 
dirty_chunks
 
std::map< MetaDataKey, size_t > num_tuples
 
std::map< MetaDataKey,
ChunkMetadataMap
chunk_metadata_map_per_fragment
 

Detailed Description

Definition at line 44 of file UpdelRoll.h.

Constructor & Destructor Documentation

UpdelRoll::~UpdelRoll ( )
inline

Definition at line 45 of file UpdelRoll.h.

References cancelUpdate(), and dirty_chunks.

45  {
46  if (dirty_chunks.size()) {
47  cancelUpdate();
48  }
49  }
void cancelUpdate()
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

Member Function Documentation

void UpdelRoll::addDirtyChunk ( std::shared_ptr< Chunk_NS::Chunk chunk,
int  fragment_id 
)

Definition at line 1486 of file UpdelStorage.cpp.

References CHECK, and Catalog_Namespace::Catalog::getDatabaseId().

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

1487  {
1489  CHECK(catalog);
1490  ChunkKey chunk_key{catalog->getDatabaseId(),
1491  chunk->getColumnDesc()->tableId,
1492  chunk->getColumnDesc()->columnId,
1493  fragment_id};
1494  dirty_chunks[chunk_key] = chunk;
1495 }
std::vector< int > ChunkKey
Definition: types.h:36
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
std::unique_lock< T > unique_lock
int getDatabaseId() const
Definition: Catalog.h:326
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:291
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void UpdelRoll::cancelUpdate ( )

Definition at line 1460 of file UpdelStorage.cpp.

References CHECK, Data_Namespace::DataMgr::free(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, and Catalog_Namespace::Catalog::setTableEpochs().

Referenced by ~UpdelRoll().

1460  {
1461  if (nullptr == catalog) {
1462  return;
1463  }
1464 
1465  // TODO: needed?
1467  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);
1468  if (is_varlen_update) {
1469  int databaseId = catalog->getDatabaseId();
1470  auto table_epochs = catalog->getTableEpochs(databaseId, logicalTableId);
1471 
1472  dirty_chunks.clear();
1473  catalog->setTableEpochs(databaseId, table_epochs);
1474  } else {
1475  const auto td = catalog->getMetadataForTable(logicalTableId);
1476  CHECK(td);
1477  if (td->persistenceLevel != memoryLevel) {
1478  for (const auto& [chunk_key, chunk] : dirty_chunks) {
1479  catalog->getDataMgr().free(chunk->getBuffer());
1480  chunk->setBuffer(nullptr);
1481  }
1482  }
1483  }
1484 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
bool is_varlen_update
Definition: UpdelRoll.h:57
std::vector< int > ChunkKey
Definition: types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:225
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:326
int logicalTableId
Definition: UpdelRoll.h:54
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3849
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void free(AbstractBuffer *buffer)
Definition: DataMgr.cpp:564
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3821

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool UpdelRoll::commitUpdate ( )

Definition at line 1402 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::Catalog::checkpoint(), Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, Catalog_Namespace::Catalog::setTableEpochsLogExceptions(), and TableDescriptor::tableId.

Referenced by StorageIOFacility::TransactionParameters::finalizeTransaction().

1402  {
1403  if (nullptr == catalog) {
1404  return false;
1405  }
1406  const auto td = catalog->getMetadataForTable(logicalTableId);
1407  CHECK(td);
1408  ChunkKey chunk_key{catalog->getDatabaseId(), td->tableId};
1409  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);
1410 
1411  // Checkpoint all shards. Otherwise, epochs can go out of sync.
1412  if (td->persistenceLevel == Data_Namespace::MemoryLevel::DISK_LEVEL) {
1413  auto table_epochs = catalog->getTableEpochs(catalog->getDatabaseId(), logicalTableId);
1414  try {
1415  // `checkpointWithAutoRollback` is not called here because, if a failure occurs,
1416  // `dirtyChunks` has to be cleared before resetting epochs
1418  } catch (...) {
1419  dirty_chunks.clear();
1421  throw;
1422  }
1423  }
1425  return true;
1426 }
std::vector< int > ChunkKey
Definition: types.h:36
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:225
void updateFragmenterAndCleanupChunks()
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:326
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:5022
int logicalTableId
Definition: UpdelRoll.h:54
#define CHECK(condition)
Definition: Logger.h:291
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3885
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3821

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > UpdelRoll::getChunkMetadata ( const MetaDataKey key,
int32_t  column_id,
Fragmenter_Namespace::FragmentInfo fragment_info 
)

Definition at line 1511 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

1514  {
1515  initializeUnsetMetadata(key.first, fragment_info);
1517  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
1518  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
1519  auto chunk_metadata_it = metadata_map_it->second.find(column_id);
1520  CHECK(chunk_metadata_it != metadata_map_it->second.end());
1521  return chunk_metadata_it->second;
1522 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
void initializeUnsetMetadata(const TableDescriptor *td, Fragmenter_Namespace::FragmentInfo &fragment_info)
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

ChunkMetadataMap UpdelRoll::getChunkMetadataMap ( const MetaDataKey key) const

Definition at line 1524 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

1524  {
1526  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
1527  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
1528  return metadata_map_it->second;
1529 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

size_t UpdelRoll::getNumTuple ( const MetaDataKey key) const

Definition at line 1531 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

1531  {
1533  auto it = num_tuples.find(key);
1534  CHECK(it != num_tuples.end());
1535  return it->second;
1536 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

void UpdelRoll::initializeUnsetMetadata ( const TableDescriptor td,
Fragmenter_Namespace::FragmentInfo fragment_info 
)
private

Definition at line 1497 of file UpdelStorage.cpp.

References Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysicalCopy(), and Fragmenter_Namespace::FragmentInfo::shadowNumTuples.

1499  {
1501  MetaDataKey key{td, &fragment_info};
1502  if (chunk_metadata_map_per_fragment.count(key) == 0) {
1504  fragment_info.getChunkMetadataMapPhysicalCopy();
1505  }
1506  if (num_tuples.count(key) == 0) {
1507  num_tuples[key] = fragment_info.shadowNumTuples;
1508  }
1509 }
ChunkMetadataMap getChunkMetadataMapPhysicalCopy() const
std::pair< const TableDescriptor *, Fragmenter_Namespace::FragmentInfo * > MetaDataKey
Definition: UpdelRoll.h:41
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
std::unique_lock< T > unique_lock
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90

+ Here is the call graph for this function:

void UpdelRoll::setNumTuple ( const MetaDataKey key,
size_t  num_tuple 
)

Definition at line 1538 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

1538  {
1540  num_tuples[key] = num_tuple;
1541 }
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
std::unique_lock< T > unique_lock
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90

+ Here is the caller graph for this function:

void UpdelRoll::stageUpdate ( )

Definition at line 1428 of file UpdelStorage.cpp.

References CHECK, CHECK_EQ, Data_Namespace::DataMgr::checkpoint(), Data_Namespace::CPU_LEVEL, Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), and Catalog_Namespace::Catalog::getDataMgr().

Referenced by TableOptimizer::vacuumFragments().

1428  {
1429  CHECK(catalog);
1430  auto db_id = catalog->getDatabaseId();
1432  auto table_id = table_descriptor->tableId;
1435  try {
1436  catalog->getDataMgr().checkpoint(db_id, table_id, memoryLevel);
1437  } catch (...) {
1438  dirty_chunks.clear();
1439  throw;
1440  }
1442 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
#define CHECK_EQ(x, y)
Definition: Logger.h:301
const TableDescriptor * table_descriptor
Definition: UpdelRoll.h:58
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:584
void updateFragmenterAndCleanupChunks()
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:326
Data_Namespace::MemoryLevel persistenceLevel
#define CHECK(condition)
Definition: Logger.h:291
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void UpdelRoll::updateFragmenterAndCleanupChunks ( )
private

Definition at line 1444 of file UpdelStorage.cpp.

References Data_Namespace::DataMgr::deleteChunksWithPrefix(), Catalog_Namespace::Catalog::getDataMgr(), and Data_Namespace::GPU_LEVEL.

1444  {
1445  // for each dirty fragment
1446  for (auto& cm : chunk_metadata_map_per_fragment) {
1447  cm.first.first->fragmenter->updateMetadata(catalog, cm.first, *this);
1448  }
1449 
1450  // flush gpu dirty chunks if update was not on gpu
1452  for (const auto& [chunk_key, chunk] : dirty_chunks) {
1455  }
1456  }
1457  dirty_chunks.clear();
1458 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:522
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

Member Data Documentation

std::map<MetaDataKey, ChunkMetadataMap> UpdelRoll::chunk_metadata_map_per_fragment
private

Definition at line 99 of file UpdelRoll.h.

heavyai::shared_mutex UpdelRoll::chunk_update_tracker_mutex
mutableprivate

Definition at line 90 of file UpdelRoll.h.

std::map<ChunkKey, std::shared_ptr<Chunk_NS::Chunk> > UpdelRoll::dirty_chunks
private

Definition at line 93 of file UpdelRoll.h.

Referenced by ~UpdelRoll().

bool UpdelRoll::is_varlen_update = false
std::map<MetaDataKey, size_t> UpdelRoll::num_tuples
private

Definition at line 96 of file UpdelRoll.h.

const TableDescriptor* UpdelRoll::table_descriptor {nullptr}

Definition at line 58 of file UpdelRoll.h.

Referenced by TableOptimizer::vacuumFragments().


The documentation for this struct was generated from the following files: