OmniSciDB  4201147b46
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
UpdelRoll Struct Reference

#include <UpdelRoll.h>

+ Collaboration diagram for UpdelRoll:

Public Member Functions

 ~UpdelRoll ()
 
void cancelUpdate ()
 
bool commitUpdate ()
 
void stageUpdate ()
 
void addDirtyChunk (std::shared_ptr< Chunk_NS::Chunk > chunk, int fragment_id)
 
std::shared_ptr< ChunkMetadatagetChunkMetadata (const MetaDataKey &key, int32_t column_id, Fragmenter_Namespace::FragmentInfo &fragment_info)
 
ChunkMetadataMap getChunkMetadataMap (const MetaDataKey &key) const
 
size_t getNumTuple (const MetaDataKey &key) const
 
void setNumTuple (const MetaDataKey &key, size_t num_tuple)
 

Public Attributes

const Catalog_Namespace::Catalogcatalog = nullptr
 
int logicalTableId
 
Data_Namespace::MemoryLevel memoryLevel {Data_Namespace::MemoryLevel::CPU_LEVEL}
 
bool is_varlen_update = false
 
const TableDescriptortable_descriptor {nullptr}
 

Private Member Functions

void updateFragmenterAndCleanupChunks ()
 
void initializeUnsetMetadata (const TableDescriptor *td, Fragmenter_Namespace::FragmentInfo &fragment_info)
 

Private Attributes

heavyai::shared_mutex chunk_update_tracker_mutex
 
std::map< ChunkKey,
std::shared_ptr
< Chunk_NS::Chunk > > 
dirty_chunks
 
std::map< MetaDataKey, size_t > num_tuples
 
std::map< MetaDataKey,
ChunkMetadataMap
chunk_metadata_map_per_fragment
 

Detailed Description

Definition at line 44 of file UpdelRoll.h.

Constructor & Destructor Documentation

UpdelRoll::~UpdelRoll ( )
inline

Definition at line 45 of file UpdelRoll.h.

References cancelUpdate(), and dirty_chunks.

45  {
46  if (dirty_chunks.size()) {
47  cancelUpdate();
48  }
49  }
void cancelUpdate()
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

Member Function Documentation

void UpdelRoll::addDirtyChunk ( std::shared_ptr< Chunk_NS::Chunk chunk,
int  fragment_id 
)

Definition at line 1485 of file UpdelStorage.cpp.

References CHECK, and Catalog_Namespace::Catalog::getDatabaseId().

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

1486  {
1488  CHECK(catalog);
1489  ChunkKey chunk_key{catalog->getDatabaseId(),
1490  chunk->getColumnDesc()->tableId,
1491  chunk->getColumnDesc()->columnId,
1492  fragment_id};
1493  dirty_chunks[chunk_key] = chunk;
1494 }
std::vector< int > ChunkKey
Definition: types.h:36
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
std::unique_lock< T > unique_lock
int getDatabaseId() const
Definition: Catalog.h:283
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void UpdelRoll::cancelUpdate ( )

Definition at line 1459 of file UpdelStorage.cpp.

References CHECK, Data_Namespace::DataMgr::free(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, and Catalog_Namespace::Catalog::setTableEpochs().

Referenced by ~UpdelRoll().

1459  {
1460  if (nullptr == catalog) {
1461  return;
1462  }
1463 
1464  // TODO: needed?
1466  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);
1467  if (is_varlen_update) {
1468  int databaseId = catalog->getDatabaseId();
1469  auto table_epochs = catalog->getTableEpochs(databaseId, logicalTableId);
1470 
1471  dirty_chunks.clear();
1472  catalog->setTableEpochs(databaseId, table_epochs);
1473  } else {
1474  const auto td = catalog->getMetadataForTable(logicalTableId);
1475  CHECK(td);
1476  if (td->persistenceLevel != memoryLevel) {
1477  for (const auto& [chunk_key, chunk] : dirty_chunks) {
1478  catalog->getDataMgr().free(chunk->getBuffer());
1479  chunk->setBuffer(nullptr);
1480  }
1481  }
1482  }
1483 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
bool is_varlen_update
Definition: UpdelRoll.h:57
std::vector< int > ChunkKey
Definition: types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:228
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:155
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:283
int logicalTableId
Definition: UpdelRoll.h:54
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3268
#define CHECK(condition)
Definition: Logger.h:222
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void free(AbstractBuffer *buffer)
Definition: DataMgr.cpp:528
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3240

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool UpdelRoll::commitUpdate ( )

Definition at line 1401 of file UpdelStorage.cpp.

References CHECK, Catalog_Namespace::Catalog::checkpoint(), Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getMetadataForTable(), Catalog_Namespace::Catalog::getTableEpochs(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getWriteLockForTable(), TableDescriptor::persistenceLevel, Catalog_Namespace::Catalog::setTableEpochsLogExceptions(), and TableDescriptor::tableId.

Referenced by StorageIOFacility::TransactionParameters::finalizeTransaction().

1401  {
1402  if (nullptr == catalog) {
1403  return false;
1404  }
1405  const auto td = catalog->getMetadataForTable(logicalTableId);
1406  CHECK(td);
1407  ChunkKey chunk_key{catalog->getDatabaseId(), td->tableId};
1408  const auto table_lock = lockmgr::TableDataLockMgr::getWriteLockForTable(chunk_key);
1409 
1410  // Checkpoint all shards. Otherwise, epochs can go out of sync.
1411  if (td->persistenceLevel == Data_Namespace::MemoryLevel::DISK_LEVEL) {
1412  auto table_epochs = catalog->getTableEpochs(catalog->getDatabaseId(), logicalTableId);
1413  try {
1414  // `checkpointWithAutoRollback` is not called here because, if a failure occurs,
1415  // `dirtyChunks` has to be cleared before resetting epochs
1417  } catch (...) {
1418  dirty_chunks.clear();
1420  throw;
1421  }
1422  }
1424  return true;
1425 }
std::vector< int > ChunkKey
Definition: types.h:36
static WriteLock getWriteLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: LockMgrImpl.h:155
void updateFragmenterAndCleanupChunks()
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:283
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4416
int logicalTableId
Definition: UpdelRoll.h:54
#define CHECK(condition)
Definition: Logger.h:222
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3304
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3240

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > UpdelRoll::getChunkMetadata ( const MetaDataKey key,
int32_t  column_id,
Fragmenter_Namespace::FragmentInfo fragment_info 
)

Definition at line 1510 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumns().

1513  {
1514  initializeUnsetMetadata(key.first, fragment_info);
1516  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
1517  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
1518  auto chunk_metadata_it = metadata_map_it->second.find(column_id);
1519  CHECK(chunk_metadata_it != metadata_map_it->second.end());
1520  return chunk_metadata_it->second;
1521 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
void initializeUnsetMetadata(const TableDescriptor *td, Fragmenter_Namespace::FragmentInfo &fragment_info)
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

ChunkMetadataMap UpdelRoll::getChunkMetadataMap ( const MetaDataKey key) const

Definition at line 1523 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

1523  {
1525  auto metadata_map_it = chunk_metadata_map_per_fragment.find(key);
1526  CHECK(metadata_map_it != chunk_metadata_map_per_fragment.end());
1527  return metadata_map_it->second;
1528 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

size_t UpdelRoll::getNumTuple ( const MetaDataKey key) const

Definition at line 1530 of file UpdelStorage.cpp.

References CHECK.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateMetadata().

1530  {
1532  auto it = num_tuples.find(key);
1533  CHECK(it != num_tuples.end());
1534  return it->second;
1535 }
std::shared_lock< T > shared_lock
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

void UpdelRoll::initializeUnsetMetadata ( const TableDescriptor td,
Fragmenter_Namespace::FragmentInfo fragment_info 
)
private

Definition at line 1496 of file UpdelStorage.cpp.

References Fragmenter_Namespace::FragmentInfo::getChunkMetadataMapPhysicalCopy(), and Fragmenter_Namespace::FragmentInfo::shadowNumTuples.

1498  {
1500  MetaDataKey key{td, &fragment_info};
1501  if (chunk_metadata_map_per_fragment.count(key) == 0) {
1503  fragment_info.getChunkMetadataMapPhysicalCopy();
1504  }
1505  if (num_tuples.count(key) == 0) {
1506  num_tuples[key] = fragment_info.shadowNumTuples;
1507  }
1508 }
ChunkMetadataMap getChunkMetadataMapPhysicalCopy() const
std::pair< const TableDescriptor *, Fragmenter_Namespace::FragmentInfo * > MetaDataKey
Definition: UpdelRoll.h:41
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
std::unique_lock< T > unique_lock
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90

+ Here is the call graph for this function:

void UpdelRoll::setNumTuple ( const MetaDataKey key,
size_t  num_tuple 
)

Definition at line 1537 of file UpdelStorage.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::compactRows().

1537  {
1539  num_tuples[key] = num_tuple;
1540 }
std::map< MetaDataKey, size_t > num_tuples
Definition: UpdelRoll.h:96
std::unique_lock< T > unique_lock
heavyai::shared_mutex chunk_update_tracker_mutex
Definition: UpdelRoll.h:90

+ Here is the caller graph for this function:

void UpdelRoll::stageUpdate ( )

Definition at line 1427 of file UpdelStorage.cpp.

References CHECK, CHECK_EQ, Data_Namespace::DataMgr::checkpoint(), Data_Namespace::CPU_LEVEL, Data_Namespace::DISK_LEVEL, Catalog_Namespace::Catalog::getDatabaseId(), and Catalog_Namespace::Catalog::getDataMgr().

Referenced by TableOptimizer::vacuumFragments().

1427  {
1428  CHECK(catalog);
1429  auto db_id = catalog->getDatabaseId();
1431  auto table_id = table_descriptor->tableId;
1434  try {
1435  catalog->getDataMgr().checkpoint(db_id, table_id, memoryLevel);
1436  } catch (...) {
1437  dirty_chunks.clear();
1438  throw;
1439  }
1441 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
#define CHECK_EQ(x, y)
Definition: Logger.h:230
const TableDescriptor * table_descriptor
Definition: UpdelRoll.h:58
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:228
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:548
void updateFragmenterAndCleanupChunks()
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
int getDatabaseId() const
Definition: Catalog.h:283
Data_Namespace::MemoryLevel persistenceLevel
#define CHECK(condition)
Definition: Logger.h:222
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void UpdelRoll::updateFragmenterAndCleanupChunks ( )
private

Definition at line 1443 of file UpdelStorage.cpp.

References Data_Namespace::DataMgr::deleteChunksWithPrefix(), Catalog_Namespace::Catalog::getDataMgr(), and Data_Namespace::GPU_LEVEL.

1443  {
1444  // for each dirty fragment
1445  for (auto& cm : chunk_metadata_map_per_fragment) {
1446  cm.first.first->fragmenter->updateMetadata(catalog, cm.first, *this);
1447  }
1448 
1449  // flush gpu dirty chunks if update was not on gpu
1451  for (const auto& [chunk_key, chunk] : dirty_chunks) {
1454  }
1455  }
1456  dirty_chunks.clear();
1457 }
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:55
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:228
std::map< MetaDataKey, ChunkMetadataMap > chunk_metadata_map_per_fragment
Definition: UpdelRoll.h:99
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:53
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:495
std::map< ChunkKey, std::shared_ptr< Chunk_NS::Chunk > > dirty_chunks
Definition: UpdelRoll.h:93

+ Here is the call graph for this function:

Member Data Documentation

std::map<MetaDataKey, ChunkMetadataMap> UpdelRoll::chunk_metadata_map_per_fragment
private

Definition at line 99 of file UpdelRoll.h.

heavyai::shared_mutex UpdelRoll::chunk_update_tracker_mutex
mutableprivate

Definition at line 90 of file UpdelRoll.h.

std::map<ChunkKey, std::shared_ptr<Chunk_NS::Chunk> > UpdelRoll::dirty_chunks
private

Definition at line 93 of file UpdelRoll.h.

Referenced by ~UpdelRoll().

bool UpdelRoll::is_varlen_update = false
std::map<MetaDataKey, size_t> UpdelRoll::num_tuples
private

Definition at line 96 of file UpdelRoll.h.

const TableDescriptor* UpdelRoll::table_descriptor {nullptr}

Definition at line 58 of file UpdelRoll.h.

Referenced by TableOptimizer::vacuumFragments().


The documentation for this struct was generated from the following files: