OmniSciDB  a667adc9c8
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ForeignStorageCache.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #pragma once
27 
28 #include "../Shared/mapd_shared_mutex.h"
31 #include "ForeignDataWrapper.h"
32 
33 class CacheTooSmallException : public std::runtime_error {
34  public:
35  CacheTooSmallException(const std::string& msg) : std::runtime_error(msg) {}
36 };
37 
38 enum class DiskCacheLevel { none, fsi, non_fsi, all };
40  std::string path;
42  size_t num_reader_threads = 0;
43  inline bool isEnabledForMutableTables() const {
46  }
47  inline bool isEnabledForFSI() const {
49  }
50  inline bool isEnabled() const { return enabled_level != DiskCacheLevel::none; }
51 };
52 
53 using namespace Data_Namespace;
54 
55 namespace foreign_storage {
56 
57 const std::string wrapper_file_name = "wrapper_metadata.json";
58 
60  public:
61  ForeignStorageCache(const DiskCacheConfig& config);
62 
71  void cacheTableChunks(const std::vector<ChunkKey>& chunk_keys);
72  void cacheChunk(const ChunkKey&, AbstractBuffer*);
73 
74  AbstractBuffer* getCachedChunkIfExists(const ChunkKey&);
75  bool isMetadataCached(const ChunkKey&) const;
76  void cacheMetadataVec(const ChunkMetadataVector&);
77  void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector&, const ChunkKey&) const;
78  bool hasCachedMetadataForKeyPrefix(const ChunkKey&) const;
79  void clearForTablePrefix(const ChunkKey&);
80  void clear();
81  std::vector<ChunkKey> getCachedChunksForKeyPrefix(const ChunkKey&) const;
82  bool recoverCacheForTable(ChunkMetadataVector&, const ChunkKey&);
83  ChunkToBufferMap getChunkBuffersForCaching(
84  const std::vector<ChunkKey>& chunk_keys) const;
85 
86  // Get a chunk buffer for writing to disk prior to metadata creation/caching
87  AbstractBuffer* getChunkBufferForPrecaching(const ChunkKey& chunk_key,
88  bool is_new_buffer);
89 
90  void deleteBufferIfExists(const ChunkKey& chunk_key);
91 
92  inline size_t getNumCachedChunks() const { return cached_chunks_.size(); }
93  inline size_t getNumCachedMetadata() const { return cached_metadata_.size(); }
94  size_t getNumChunksAdded() const { return num_chunks_added_; }
95  size_t getNumMetadataAdded() const { return num_metadata_added_; }
96 
97  // Useful for debugging.
98  std::string dumpCachedChunkEntries() const;
99  std::string dumpCachedMetadataEntries() const;
100 
101  inline std::string getCacheDirectory() const {
102  return caching_file_mgr_->getFileMgrBasePath();
103  }
104 
105  inline std::string getCacheDirectoryForTable(int db_id, int tb_id) const {
106  return caching_file_mgr_->getOrAddTableDir(db_id, tb_id);
107  }
108 
109  void cacheMetadataWithFragIdGreaterOrEqualTo(const ChunkMetadataVector& metadata_vec,
110  const int frag_id);
111  void evictThenEraseChunk(const ChunkKey&);
112 
113  inline uint64_t getSpaceReservedByTable(int db_id, int tb_id) const {
114  return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
115  }
116 
117  private:
118  // These methods are private and assume locks are already acquired when called.
119  std::set<ChunkKey>::iterator eraseChunk(const std::set<ChunkKey>::iterator&);
120  void eraseChunk(const ChunkKey& chunk_key);
121  std::set<ChunkKey>::iterator evictChunkByIterator(
122  const std::set<ChunkKey>::iterator& chunk_it);
123  void evictThenEraseChunkUnlocked(const ChunkKey&);
124  void validatePath(const std::string&) const;
125 
126  // Underlying storage is handled by a CachingFileMgr unique to the cache.
127  std::unique_ptr<File_Namespace::CachingFileMgr> caching_file_mgr_;
128 
129  // Keeps tracks of which Chunks/ChunkMetadata are cached.
130  std::set<ChunkKey> cached_chunks_;
131  std::set<ChunkKey> cached_metadata_;
132 
133  // Keeps tracks of how many times we cache chunks or metadata for testing purposes.
136 
137  // Separate mutexes for chunks/metadata.
140 }; // ForeignStorageCache
141 } // namespace foreign_storage
DiskCacheLevel
DiskCacheLevel enabled_level
std::vector< int > ChunkKey
Definition: types.h:37
bool isEnabledForMutableTables() const
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
uint64_t getSpaceReservedByTable(int db_id, int tb_id) const
std::shared_timed_mutex mapd_shared_mutex
bool isEnabledForFSI() const
const std::string wrapper_file_name
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
CacheTooSmallException(const std::string &msg)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
std::string getCacheDirectoryForTable(int db_id, int tb_id) const
bool isEnabled() const