OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
UpdelRoll Struct Reference

#include <UpdelRoll.h>

+ Collaboration diagram for UpdelRoll:

Public Member Functions

 ~UpdelRoll ()
 
void cancelUpdate ()
 
bool commitUpdate ()
 
void stageUpdate ()
 
void addDirtyChunk (std::shared_ptr< Chunk_NS::Chunk > chunk, int fragment_id)
 
std::shared_ptr< ChunkMetadatagetChunkMetadata (const MetaDataKey &key, int32_t column_id, Fragmenter_Namespace::FragmentInfo &fragment_info)
 
ChunkMetadataMap getChunkMetadataMap (const MetaDataKey &key) const
 
size_t getNumTuple (const MetaDataKey &key) const
 
void setNumTuple (const MetaDataKey &key, size_t num_tuple)
 

Public Attributes

const Catalog_Namespace::Catalogcatalog = nullptr
 
int logicalTableId
 
Data_Namespace::MemoryLevel memoryLevel {Data_Namespace::MemoryLevel::CPU_LEVEL}
 
bool is_varlen_update = false
 
const TableDescriptortable_descriptor {nullptr}
 

Private Member Functions

void updateFragmenterAndCleanupChunks ()
 
void initializeUnsetMetadata (const TableDescriptor *td, Fragmenter_Namespace::FragmentInfo &fragment_info)
 

Private Attributes

heavyai::shared_mutex chunk_update_tracker_mutex
 
std::map< ChunkKey,
std::shared_ptr
< Chunk_NS::Chunk > > 
dirty_chunks
 
std::map< MetaDataKey, size_t > num_tuples
 
std::map< MetaDataKey,
ChunkMetadataMap
chunk_metadata_map_per_fragment
 

Detailed Description

Definition at line 44 of file UpdelRoll.h.

Constructor & Destructor Documentation

UpdelRoll::~UpdelRoll ( )
inline

Definition at line 45 of file UpdelRoll.h.

References cancelUpdate(), and dirty_chunks.

45  {
46  if (dirty_chunks.size()) {
47  cancelUpdate();
48  }
49  }
void cancelUpdate()
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

Member Function Documentation

void UpdelRoll::addDirtyChunk ( std::shared_ptr< Chunk_NS::Chunk chunk,
int  fragment_id 
)

Definition at line 1489 of file UpdelStorage.cpp.

References CHECK, and Catalog_Namespace::Catalog::getDatabaseId().

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

1490  {
1492  CHECK(catalog);
1493  ChunkKey chunk_key{catalog->getDatabaseId(),
1494  chunk->getColumnDesc()->tableId,
1495  chunk->getColumnDesc()->columnId,
1496  fragment_id};
1497  dirty_chunks[chunk_key] = chunk;
1498 }
std::vector< int > ChunkKey
Definition: types.h:36
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
std::unique_lock< T > unique_lock
int getDatabaseId() const
Definition: Catalog.h:298
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void UpdelRoll::cancelUpdate ( )

Definition at line 1463 of file UpdelStorage.cpp.

References CHECK, Data_Namespace::DataMgr::free(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, and Catalog_Namespace::Catalog::setTableEpochs().

Referenced by ~UpdelRoll().

1463  {
1464  if (nullptr == catalog) {
1465  return;
1466  }
1467 
1468  // TODO: needed?
1470  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);
1471  if (is_varlen_update) {
1472  int databaseId = catalog->getDatabaseId();
1473  auto table_epochs = catalog->getTableEpochs(databaseId, logicalTableId);
1474 
1475  dirty_chunks.clear();
1476  catalog->setTableEpochs(databaseId, table_epochs);
1477  } else {
1478  const auto td = catalog->getMetadataForTable(logicalTableId);
1479  CHECK(td);
1480  if (td->persistenceLevel != memoryLevel) {
1481  for (const auto& [chunk_key, chunk] : dirty_chunks) {
1482  catalog->getDataMgr().free(chunk->getBuffer());
1483  chunk->setBuffer(nullptr);
1484  }
1485  }
1486  }
1487 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
bool is_varlen_update
Definition: UpdelRoll.h:57
std::vector< int > ChunkKey
Definition: types.h:36
static WriteLock getWriteLockForTable(Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:225
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:243
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:298
int logicalTableId
Definition: UpdelRoll.h:54
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3613
#define CHECK(condition)
Definition: Logger.h:222
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void free(AbstractBuffer *buffer)
Definition: DataMgr.cpp:525
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3585

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool UpdelRoll::commitUpdate ( )

Definition at line 1405 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::Catalog::checkpoint(), Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, Catalog_Namespace::Catalog::setTableEpochsLogExceptions(), and TableDescriptor::tableId.

Referenced by StorageIOFacility::TransactionParameters::finalizeTransaction().

1405  {
1406  if (nullptr == catalog) {
1407  return false;
1408  }
1409  const auto td = catalog->getMetadataForTable(logicalTableId);
1410  CHECK(td);
1411  ChunkKey chunk_key{catalog->getDatabaseId(), td->tableId};
1412  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);
1413 
1414  // Checkpoint all shards. Otherwise, epochs can go out of sync.
1415  if (td->persistenceLevel == Data_Namespace::MemoryLevel::DISK_LEVEL) {
1416  auto table_epochs = catalog->getTableEpochs(catalog->getDatabaseId(), logicalTableId);
1417  try {
1418  // `checkpointWithAutoRollback` is not called here because, if a failure occurs,
1419  // `dirtyChunks` has to be cleared before resetting epochs
1421  } catch (...) {
1422  dirty_chunks.clear();
1424  throw;
1425  }
1426  }
1428  return true;
1429 }
std::vector< int > ChunkKey
Definition: types.h:36
static WriteLock getWriteLockForTable(Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:225
void updateFragmenterAndCleanupChunks()
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:298
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4783
int logicalTableId
Definition: UpdelRoll.h:54
#define CHECK(condition)
Definition: Logger.h:222
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3649
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3585

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > UpdelRoll::getChunkMetadata ( const MetaDataKey key,
int32_t  column_id,
Fragmenter_Namespace::FragmentInfo fragment_info 
)

Definition at line 1514 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

1517  {
1518  initializeUnsetMetadata(key.first, fragment_info);
1520  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
1521  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
1522  auto chunk_metadata_it = metadata_map_it->second.find(column_id);
1523  CHECK(chunk_metadata_it != metadata_map_it->second.end());
1524  return chunk_metadata_it->second;
1525 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
void initializeUnsetMetadata(const TableDescriptor *td, Fragmenter_Namespace::FragmentInfo &fragment_info)
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

ChunkMetadataMap UpdelRoll::getChunkMetadataMap ( const MetaDataKey key) const

Definition at line 1527 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

1527  {
1529  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
1530  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
1531  return metadata_map_it->second;
1532 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

size_t UpdelRoll::getNumTuple ( const MetaDataKey key) const

Definition at line 1534 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

1534  {
1536  auto it = num_tuples.find(key);
1537  CHECK(it != num_tuples.end());
1538  return it->second;
1539 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

void UpdelRoll::initializeUnsetMetadata ( const TableDescriptor td,
Fragmenter_Namespace::FragmentInfo fragment_info 
)
private

Definition at line 1500 of file UpdelStorage.cpp.

References Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysicalCopy(), and Fragmenter_Namespace::FragmentInfo::shadowNumTuples.

1502  {
1504  MetaDataKey key{td, &fragment_info};
1505  if (chunk_metadata_map_per_fragment.count(key) == 0) {
1507  fragment_info.getChunkMetadataMapPhysicalCopy();
1508  }
1509  if (num_tuples.count(key) == 0) {
1510  num_tuples[key] = fragment_info.shadowNumTuples;
1511  }
1512 }
ChunkMetadataMap getChunkMetadataMapPhysicalCopy() const
std::pair< const TableDescriptor *, Fragmenter_Namespace::FragmentInfo * > MetaDataKey
Definition: UpdelRoll.h:41
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
std::unique_lock< T > unique_lock
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90

+ Here is the call graph for this function:

void UpdelRoll::setNumTuple ( const MetaDataKey key,
size_t  num_tuple 
)

Definition at line 1541 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

1541  {
1543  num_tuples[key] = num_tuple;
1544 }
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
std::unique_lock< T > unique_lock
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90

+ Here is the caller graph for this function:

void UpdelRoll::stageUpdate ( )

Definition at line 1431 of file UpdelStorage.cpp.

References CHECK, CHECK_EQ, Data_Namespace::DataMgr::checkpoint(), Data_Namespace::CPU_LEVEL, Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), and Catalog_Namespace::Catalog::getDataMgr().

Referenced by TableOptimizer::vacuumFragments().

1431  {
1432  CHECK(catalog);
1433  auto db_id = catalog->getDatabaseId();
1435  auto table_id = table_descriptor->tableId;
1438  try {
1439  catalog->getDataMgr().checkpoint(db_id, table_id, memoryLevel);
1440  } catch (...) {
1441  dirty_chunks.clear();
1442  throw;
1443  }
1445 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
#define CHECK_EQ(x, y)
Definition: Logger.h:230
const TableDescriptor * table_descriptor
Definition: UpdelRoll.h:58
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:243
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:545
void updateFragmenterAndCleanupChunks()
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:298
Data_Namespace::MemoryLevel persistenceLevel
#define CHECK(condition)
Definition: Logger.h:222
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void UpdelRoll::updateFragmenterAndCleanupChunks ( )
private

Definition at line 1447 of file UpdelStorage.cpp.

References Data_Namespace::DataMgr::deleteChunksWithPrefix(), Catalog_Namespace::Catalog::getDataMgr(), and Data_Namespace::GPU_LEVEL.

1447  {
1448  // for each dirty fragment
1449  for (auto& cm : chunk_metadata_map_per_fragment) {
1450  cm.first.first->fragmenter->updateMetadata(catalog, cm.first, *this);
1451  }
1452 
1453  // flush gpu dirty chunks if update was not on gpu
1455  for (const auto& [chunk_key, chunk] : dirty_chunks) {
1458  }
1459  }
1460  dirty_chunks.clear();
1461 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:243
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:492
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

Member Data Documentation

std::map<MetaDataKey, ChunkMetadataMap> UpdelRoll::chunk_metadata_map_per_fragment
private

Definition at line 99 of file UpdelRoll.h.

heavyai::shared_mutex UpdelRoll::chunk_update_tracker_mutex
mutableprivate

Definition at line 90 of file UpdelRoll.h.

std::map<ChunkKey, std::shared_ptr<Chunk_NS::Chunk> > UpdelRoll::dirty_chunks
private

Definition at line 93 of file UpdelRoll.h.

Referenced by ~UpdelRoll().

bool UpdelRoll::is_varlen_update = false
std::map<MetaDataKey, size_t> UpdelRoll::num_tuples
private

Definition at line 96 of file UpdelRoll.h.

const TableDescriptor* UpdelRoll::table_descriptor {nullptr}

Definition at line 58 of file UpdelRoll.h.

Referenced by TableOptimizer::vacuumFragments().


The documentation for this struct was generated from the following files: