OmniSciDB  91042dcc5b
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
HashingSchemeRecycler Class Reference

#include <HashingSchemeRecycler.h>

+ Inheritance diagram for HashingSchemeRecycler:
+ Collaboration diagram for HashingSchemeRecycler:

Public Member Functions

 HashingSchemeRecycler ()
 
 std::numeric_limits ()
 
 std::numeric_limits ()
 
void putItemToCache (QueryPlanHash key, std::optional< HashType > item, CacheItemType item_type, DeviceIdentifier device_identifier, size_t item_size, size_t compute_time, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override
 
void initCache () override
 
void clearCache () override
 
void markCachedItemAsDirty (size_t table_key, std::unordered_set< QueryPlanHash > &key_set, CacheItemType item_type, DeviceIdentifier device_identifier) override
 
std::string toString () const override
 
- Public Member Functions inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
 DataRecycler (const std::vector< CacheItemType > &item_types, size_t total_cache_size, size_t max_item_size, int num_gpus)
 
virtual ~DataRecycler ()=default
 
virtual std::optional< HashTypegetItemFromCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::optional< EMPTY_META_INFO > meta_info=std::nullopt)=0
 
void markCachedItemAsDirtyImpl (QueryPlanHash key, CachedItemContainer &m) const
 
bool isCachedItemDirty (QueryPlanHash key, CachedItemContainer &m) const
 
std::shared_ptr
< CachedItemContainer
getCachedItemContainer (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
std::optional< CachedItem
< std::optional< HashType >
, EMPTY_META_INFO > > 
getCachedItemWithoutConsideringMetaInfo (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, CachedItemContainer &m, std::lock_guard< std::mutex > &lock)
 
size_t getCurrentNumCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
size_t getCurrentNumDirtyCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
size_t getCurrentNumCleanCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
size_t getCurrentCacheSizeForDevice (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
std::shared_ptr< CacheItemMetricgetCachedItemMetric (CacheItemType item_type, DeviceIdentifier device_identifier, QueryPlanHash key) const
 
void setTotalCacheSize (CacheItemType item_type, size_t new_total_cache_size)
 
void setMaxCacheItemSize (CacheItemType item_type, size_t new_max_cache_item_size)
 

Private Member Functions

bool hasItemInCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) const override
 
void removeItemFromCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override
 
void cleanupCacheForInsertion (CacheItemType item_type, DeviceIdentifier device_identifier, size_t required_size, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override
 

Additional Inherited Members

- Public Types inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
using CachedItemContainer = std::vector< CachedItem< std::optional< HashType >, EMPTY_META_INFO >>
 
using PerDeviceCacheItemContainer = std::unordered_map< DeviceIdentifier, std::shared_ptr< CachedItemContainer >>
 
using PerTypeCacheItemContainer = std::unordered_map< CacheItemType, std::shared_ptr< PerDeviceCacheItemContainer >>
 
using PerTypeCacheMetricTracker = std::unordered_map< CacheItemType, CacheMetricTracker >
 
- Protected Member Functions inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
void removeCachedItemFromBeginning (CacheItemType item_type, DeviceIdentifier device_identifier, int offset)
 
void sortCacheContainerByQueryMetric (CacheItemType item_type, DeviceIdentifier device_identifier)
 
std::mutex & getCacheLock () const
 
CacheMetricTrackergetMetricTracker (CacheItemType item_type)
 
CacheMetricTracker constgetMetricTracker (CacheItemType item_type) const
 
std::unordered_set
< CacheItemType > const
getCacheItemType () const
 
PerTypeCacheItemContainer constgetItemCache () const
 

Detailed Description

Definition at line 24 of file HashingSchemeRecycler.h.

Constructor & Destructor Documentation

HashingSchemeRecycler::HashingSchemeRecycler ( )
inline

Definition at line 30 of file HashingSchemeRecycler.h.

References HT_HASHING_SCHEME.

DataRecycler(const std::vector< CacheItemType > &item_types, size_t total_cache_size, size_t max_item_size, int num_gpus)
Definition: DataRecycler.h:410

Member Function Documentation

void HashingSchemeRecycler::cleanupCacheForInsertion ( CacheItemType  item_type,
DeviceIdentifier  device_identifier,
size_t  required_size,
std::lock_guard< std::mutex > &  lock,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
)
inlineoverrideprivatevirtual
void HashingSchemeRecycler::clearCache ( )
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 87 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), HT_HASHING_SCHEME, and LAYOUT_CACHE_DEVICE_IDENTIFIER.

87  {
88  std::lock_guard<std::mutex> lock(getCacheLock());
89  auto layout_cache_container = getCachedItemContainer(CacheItemType::HT_HASHING_SCHEME,
91  layout_cache_container->clear();
92 }
constexpr DeviceIdentifier LAYOUT_CACHE_DEVICE_IDENTIFIER
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:480

+ Here is the call graph for this function:

bool HashingSchemeRecycler::hasItemInCache ( QueryPlanHash  key,
CacheItemType  item_type,
DeviceIdentifier  device_identifier,
std::lock_guard< std::mutex > &  lock,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
) const
overrideprivatevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 125 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, EMPTY_HASHED_PLAN_DAG_KEY, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), and HT_HASHING_SCHEME.

130  {
132  key == EMPTY_HASHED_PLAN_DAG_KEY) {
133  return false;
134  }
136  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
137  return std::any_of(layout_cache->begin(),
138  layout_cache->end(),
139  [&key](const auto& cached_item) { return cached_item.key == key; });
140 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
constexpr QueryPlanHash EMPTY_HASHED_PLAN_DAG_KEY
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:480
bool g_enable_data_recycler
Definition: Execute.cpp:141
bool g_use_hashtable_cache
Definition: Execute.cpp:142

+ Here is the call graph for this function:

void HashingSchemeRecycler::initCache ( )
inlineoverridevirtual
void HashingSchemeRecycler::markCachedItemAsDirty ( size_t  table_key,
std::unordered_set< QueryPlanHash > &  key_set,
CacheItemType  item_type,
DeviceIdentifier  device_identifier 
)
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 94 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), HT_HASHING_SCHEME, and DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::markCachedItemAsDirtyImpl().

98  {
99  if (!g_enable_data_recycler || !g_use_hashtable_cache || key_set.empty()) {
100  return;
101  }
103  std::lock_guard<std::mutex> lock(getCacheLock());
104  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
105  for (auto key : key_set) {
106  markCachedItemAsDirtyImpl(key, *layout_cache);
107  }
108 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:480
void markCachedItemAsDirtyImpl(QueryPlanHash key, CachedItemContainer &m) const
Definition: DataRecycler.h:456
bool g_enable_data_recycler
Definition: Execute.cpp:141
bool g_use_hashtable_cache
Definition: Execute.cpp:142

+ Here is the call graph for this function:

void HashingSchemeRecycler::putItemToCache ( QueryPlanHash  key,
std::optional< HashType item,
CacheItemType  item_type,
DeviceIdentifier  device_identifier,
size_t  item_size,
size_t  compute_time,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
)
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 45 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, EMPTY_HASHED_PLAN_DAG_KEY, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemWithoutConsideringMetaInfo(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), DataRecyclerUtil::getDeviceIdentifierString(), HT_HASHING_SCHEME, DataRecyclerUtil::toStringCacheItemType(), and VLOG.

51  {
54  return;
55  }
57  std::lock_guard<std::mutex> lock(getCacheLock());
58  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
59  auto candidate_layout = getCachedItemWithoutConsideringMetaInfo(
60  key, item_type, device_identifier, *layout_cache, lock);
61  if (candidate_layout) {
62  return;
63  }
64  layout_cache->emplace_back(key, item, nullptr, meta_info);
65  VLOG(1) << "[" << DataRecyclerUtil::toStringCacheItemType(item_type) << ", "
67  << "] Put hashtable layout to cache";
68  return;
69 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
static std::string getDeviceIdentifierString(DeviceIdentifier device_identifier)
Definition: DataRecycler.h:135
static std::string_view toStringCacheItemType(CacheItemType item_type)
Definition: DataRecycler.h:128
std::optional< CachedItem< std::optional< HashType >, EMPTY_META_INFO > > getCachedItemWithoutConsideringMetaInfo(QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, CachedItemContainer &m, std::lock_guard< std::mutex > &lock)
Definition: DataRecycler.h:495
constexpr QueryPlanHash EMPTY_HASHED_PLAN_DAG_KEY
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:480
bool g_enable_data_recycler
Definition: Execute.cpp:141
bool g_use_hashtable_cache
Definition: Execute.cpp:142
#define VLOG(n)
Definition: Logger.h:305

+ Here is the call graph for this function:

void HashingSchemeRecycler::removeItemFromCache ( QueryPlanHash  key,
CacheItemType  item_type,
DeviceIdentifier  device_identifier,
std::lock_guard< std::mutex > &  lock,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
)
overrideprivatevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 71 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer().

76  {
77  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
78  auto filter = [key](auto const& item) { return item.key == key; };
79  auto itr = std::find_if(layout_cache->cbegin(), layout_cache->cend(), filter);
80  if (itr == layout_cache->cend()) {
81  return;
82  } else {
83  layout_cache->erase(itr);
84  }
85 }
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:480

+ Here is the call graph for this function:

HashingSchemeRecycler::std::numeric_limits ( )
HashingSchemeRecycler::std::numeric_limits ( )
std::string HashingSchemeRecycler::toString ( ) const
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 110 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecyclerUtil::getDeviceIdentifierString(), HashJoin::getHashTypeString(), HT_HASHING_SCHEME, and LAYOUT_CACHE_DEVICE_IDENTIFIER.

110  {
111  auto layout_cache_container = getCachedItemContainer(CacheItemType::HT_HASHING_SCHEME,
113  std::ostringstream oss;
114  oss << "Hashing scheme cache:\n";
115  oss << "Device: "
117  << "\n";
118  for (auto& kv : *layout_cache_container) {
119  oss << "\tkey: " << kv.key
120  << ", layout: " << HashJoin::getHashTypeString(*kv.cached_item) << "\n";
121  }
122  return oss.str();
123 }
static std::string getDeviceIdentifierString(DeviceIdentifier device_identifier)
Definition: DataRecycler.h:135
constexpr DeviceIdentifier LAYOUT_CACHE_DEVICE_IDENTIFIER
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:480
static std::string getHashTypeString(HashType ht) noexcept
Definition: HashJoin.h:134

+ Here is the call graph for this function:


The documentation for this class was generated from the following files: