OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
HashingSchemeRecycler Class Reference

#include <HashingSchemeRecycler.h>

+ Inheritance diagram for HashingSchemeRecycler:
+ Collaboration diagram for HashingSchemeRecycler:

Public Member Functions

 HashingSchemeRecycler ()
 
 std::numeric_limits ()
 
 std::numeric_limits ()
 
void putItemToCache (QueryPlanHash key, std::optional< HashType > item, CacheItemType item_type, DeviceIdentifier device_identifier, size_t item_size, size_t compute_time, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override
 
void initCache () override
 
void clearCache () override
 
void markCachedItemAsDirty (size_t table_key, std::unordered_set< QueryPlanHash > &key_set, CacheItemType item_type, DeviceIdentifier device_identifier) override
 
std::string toString () const override
 
- Public Member Functions inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
 DataRecycler (const std::vector< CacheItemType > &item_types, size_t total_cache_size, size_t max_item_size, int num_gpus)
 
virtual ~DataRecycler ()=default
 
virtual std::optional< HashTypegetItemFromCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::optional< EMPTY_META_INFO > meta_info=std::nullopt)=0
 
void markCachedItemAsDirtyImpl (QueryPlanHash key, CachedItemContainer &m) const
 
bool isCachedItemDirty (QueryPlanHash key, CachedItemContainer &m) const
 
std::shared_ptr
< CachedItemContainer
getCachedItemContainer (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
std::optional< CachedItem
< std::optional< HashType >
, EMPTY_META_INFO > > 
getCachedItemWithoutConsideringMetaInfo (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, CachedItemContainer &m, std::lock_guard< std::mutex > &lock)
 
size_t getCurrentNumCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
size_t getCurrentNumDirtyCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
size_t getCurrentNumCleanCachedItems (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
size_t getCurrentCacheSizeForDevice (CacheItemType item_type, DeviceIdentifier device_identifier) const
 
std::shared_ptr< CacheItemMetricgetCachedItemMetric (CacheItemType item_type, DeviceIdentifier device_identifier, QueryPlanHash key) const
 
void setTotalCacheSize (CacheItemType item_type, size_t new_total_cache_size)
 
void setMaxCacheItemSize (CacheItemType item_type, size_t new_max_cache_item_size)
 

Private Member Functions

bool hasItemInCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) const override
 
void removeItemFromCache (QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override
 
void cleanupCacheForInsertion (CacheItemType item_type, DeviceIdentifier device_identifier, size_t required_size, std::lock_guard< std::mutex > &lock, std::optional< EMPTY_META_INFO > meta_info=std::nullopt) override
 

Additional Inherited Members

- Public Types inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
using CachedItemContainer = std::vector< CachedItem< std::optional< HashType >, EMPTY_META_INFO >>
 
using PerDeviceCacheItemContainer = std::unordered_map< DeviceIdentifier, std::shared_ptr< CachedItemContainer >>
 
using PerTypeCacheItemContainer = std::unordered_map< CacheItemType, std::shared_ptr< PerDeviceCacheItemContainer >>
 
using PerTypeCacheMetricTracker = std::unordered_map< CacheItemType, CacheMetricTracker >
 
- Protected Member Functions inherited from DataRecycler< std::optional< HashType >, EMPTY_META_INFO >
void removeCachedItemFromBeginning (CacheItemType item_type, DeviceIdentifier device_identifier, int offset)
 
void sortCacheContainerByQueryMetric (CacheItemType item_type, DeviceIdentifier device_identifier)
 
std::mutex & getCacheLock () const
 
CacheMetricTrackergetMetricTracker (CacheItemType item_type)
 
CacheMetricTracker const & getMetricTracker (CacheItemType item_type) const
 
std::unordered_set
< CacheItemType > const & 
getCacheItemType () const
 
PerTypeCacheItemContainer const & getItemCache () const
 

Detailed Description

Definition at line 24 of file HashingSchemeRecycler.h.

Constructor & Destructor Documentation

HashingSchemeRecycler::HashingSchemeRecycler ( )
inline

Definition at line 30 of file HashingSchemeRecycler.h.

References HT_HASHING_SCHEME.

DataRecycler(const std::vector< CacheItemType > &item_types, size_t total_cache_size, size_t max_item_size, int num_gpus)
Definition: DataRecycler.h:460

Member Function Documentation

void HashingSchemeRecycler::cleanupCacheForInsertion ( CacheItemType  item_type,
DeviceIdentifier  device_identifier,
size_t  required_size,
std::lock_guard< std::mutex > &  lock,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
)
inlineoverrideprivatevirtual
void HashingSchemeRecycler::clearCache ( )
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 90 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), DataRecyclerUtil::getDeviceIdentifierString(), HT_HASHING_SCHEME, LAYOUT_CACHE_DEVICE_IDENTIFIER, and VLOG.

90  {
91  std::lock_guard<std::mutex> lock(getCacheLock());
92  auto layout_cache_container = getCachedItemContainer(CacheItemType::HT_HASHING_SCHEME,
94  if (!layout_cache_container->empty()) {
95  VLOG(1) << "[" << CacheItemType::HT_HASHING_SCHEME << ", "
97  << "] clear cache (# items: " << layout_cache_container->size() << ")";
98  layout_cache_container->clear();
99  }
100 }
static std::string getDeviceIdentifierString(DeviceIdentifier device_identifier)
Definition: DataRecycler.h:138
constexpr DeviceIdentifier LAYOUT_CACHE_DEVICE_IDENTIFIER
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:530
#define VLOG(n)
Definition: Logger.h:316

+ Here is the call graph for this function:

bool HashingSchemeRecycler::hasItemInCache ( QueryPlanHash  key,
CacheItemType  item_type,
DeviceIdentifier  device_identifier,
std::lock_guard< std::mutex > &  lock,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
) const
overrideprivatevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 133 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, EMPTY_HASHED_PLAN_DAG_KEY, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), and HT_HASHING_SCHEME.

138  {
140  key == EMPTY_HASHED_PLAN_DAG_KEY) {
141  return false;
142  }
144  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
145  return std::any_of(layout_cache->begin(),
146  layout_cache->end(),
147  [&key](const auto& cached_item) { return cached_item.key == key; });
148 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
constexpr QueryPlanHash EMPTY_HASHED_PLAN_DAG_KEY
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:530
bool g_enable_data_recycler
Definition: Execute.cpp:146
bool g_use_hashtable_cache
Definition: Execute.cpp:147

+ Here is the call graph for this function:

void HashingSchemeRecycler::initCache ( )
inlineoverridevirtual
void HashingSchemeRecycler::markCachedItemAsDirty ( size_t  table_key,
std::unordered_set< QueryPlanHash > &  key_set,
CacheItemType  item_type,
DeviceIdentifier  device_identifier 
)
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 102 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), HT_HASHING_SCHEME, and DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::markCachedItemAsDirtyImpl().

106  {
107  if (!g_enable_data_recycler || !g_use_hashtable_cache || key_set.empty()) {
108  return;
109  }
111  std::lock_guard<std::mutex> lock(getCacheLock());
112  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
113  for (auto key : key_set) {
114  markCachedItemAsDirtyImpl(key, *layout_cache);
115  }
116 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:530
void markCachedItemAsDirtyImpl(QueryPlanHash key, CachedItemContainer &m) const
Definition: DataRecycler.h:506
bool g_enable_data_recycler
Definition: Execute.cpp:146
bool g_use_hashtable_cache
Definition: Execute.cpp:147

+ Here is the call graph for this function:

void HashingSchemeRecycler::putItemToCache ( QueryPlanHash  key,
std::optional< HashType item,
CacheItemType  item_type,
DeviceIdentifier  device_identifier,
size_t  item_size,
size_t  compute_time,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
)
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 45 of file HashingSchemeRecycler.cpp.

References CHECK_EQ, EMPTY_HASHED_PLAN_DAG_KEY, g_enable_data_recycler, g_use_hashtable_cache, DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemWithoutConsideringMetaInfo(), DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCacheLock(), DataRecyclerUtil::getDeviceIdentifierString(), HT_HASHING_SCHEME, and VLOG.

51  {
54  return;
55  }
57  std::lock_guard<std::mutex> lock(getCacheLock());
58  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
59  auto candidate_layout = getCachedItemWithoutConsideringMetaInfo(
60  key, item_type, device_identifier, *layout_cache, lock);
61  if (candidate_layout) {
62  return;
63  }
64  layout_cache->emplace_back(key, item, nullptr, meta_info);
65  VLOG(1) << "[" << item_type << ", "
67  << "] Put hashtable layout to cache (key: " << key << ")";
68  return;
69 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
static std::string getDeviceIdentifierString(DeviceIdentifier device_identifier)
Definition: DataRecycler.h:138
std::optional< CachedItem< std::optional< HashType >, EMPTY_META_INFO > > getCachedItemWithoutConsideringMetaInfo(QueryPlanHash key, CacheItemType item_type, DeviceIdentifier device_identifier, CachedItemContainer &m, std::lock_guard< std::mutex > &lock)
Definition: DataRecycler.h:545
constexpr QueryPlanHash EMPTY_HASHED_PLAN_DAG_KEY
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:530
bool g_enable_data_recycler
Definition: Execute.cpp:146
bool g_use_hashtable_cache
Definition: Execute.cpp:147
#define VLOG(n)
Definition: Logger.h:316

+ Here is the call graph for this function:

void HashingSchemeRecycler::removeItemFromCache ( QueryPlanHash  key,
CacheItemType  item_type,
DeviceIdentifier  device_identifier,
std::lock_guard< std::mutex > &  lock,
std::optional< EMPTY_META_INFO meta_info = std::nullopt 
)
overrideprivatevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 71 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecyclerUtil::getDeviceIdentifierString(), and VLOG.

76  {
77  auto layout_cache = getCachedItemContainer(item_type, device_identifier);
78  auto filter = [key](auto const& item) { return item.key == key; };
79  auto itr = std::find_if(layout_cache->cbegin(), layout_cache->cend(), filter);
80  if (itr == layout_cache->cend()) {
81  return;
82  } else {
83  VLOG(1) << "[" << item_type << ", "
85  << "] remove cached item from cache (key: " << key << ")";
86  layout_cache->erase(itr);
87  }
88 }
static std::string getDeviceIdentifierString(DeviceIdentifier device_identifier)
Definition: DataRecycler.h:138
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:530
#define VLOG(n)
Definition: Logger.h:316

+ Here is the call graph for this function:

HashingSchemeRecycler::std::numeric_limits ( )
HashingSchemeRecycler::std::numeric_limits ( )
std::string HashingSchemeRecycler::toString ( ) const
overridevirtual

Implements DataRecycler< std::optional< HashType >, EMPTY_META_INFO >.

Definition at line 118 of file HashingSchemeRecycler.cpp.

References DataRecycler< std::optional< HashType >, EMPTY_META_INFO >::getCachedItemContainer(), DataRecyclerUtil::getDeviceIdentifierString(), HashJoin::getHashTypeString(), HT_HASHING_SCHEME, and LAYOUT_CACHE_DEVICE_IDENTIFIER.

118  {
119  auto layout_cache_container = getCachedItemContainer(CacheItemType::HT_HASHING_SCHEME,
121  std::ostringstream oss;
122  oss << "Hashing scheme cache:\n";
123  oss << "Device: "
125  << "\n";
126  for (auto& kv : *layout_cache_container) {
127  oss << "\tkey: " << kv.key
128  << ", layout: " << HashJoin::getHashTypeString(*kv.cached_item) << "\n";
129  }
130  return oss.str();
131 }
static std::string getDeviceIdentifierString(DeviceIdentifier device_identifier)
Definition: DataRecycler.h:138
constexpr DeviceIdentifier LAYOUT_CACHE_DEVICE_IDENTIFIER
std::shared_ptr< CachedItemContainer > getCachedItemContainer(CacheItemType item_type, DeviceIdentifier device_identifier) const
Definition: DataRecycler.h:530
static std::string getHashTypeString(HashType ht) noexcept
Definition: HashJoin.h:154

+ Here is the call graph for this function:


The documentation for this class was generated from the following files: