OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ForeignStorageCache.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #pragma once
27 
28 #include "../Shared/mapd_shared_mutex.h"
31 #include "ForeignDataWrapper.h"
32 
33 class CacheTooSmallException : public std::runtime_error {
34  public:
35  CacheTooSmallException(const std::string& msg) : std::runtime_error(msg) {}
36 };
37 
38 enum class DiskCacheLevel { none, fsi, non_fsi, all };
40  std::string path;
42  size_t num_reader_threads = 0;
44  inline bool isEnabledForMutableTables() const {
47  }
48  inline bool isEnabledForFSI() const {
50  }
51  inline bool isEnabled() const { return enabled_level != DiskCacheLevel::none; }
52 };
53 
54 using namespace Data_Namespace;
55 
56 namespace foreign_storage {
57 
58 const std::string wrapper_file_name = "wrapper_metadata.json";
59 
61  public:
62  ForeignStorageCache(const DiskCacheConfig& config);
63 
72  void cacheTableChunks(const std::vector<ChunkKey>& chunk_keys);
73  void cacheChunk(const ChunkKey&, AbstractBuffer*);
74 
75  File_Namespace::FileBuffer* getCachedChunkIfExists(const ChunkKey&);
76  bool isMetadataCached(const ChunkKey&) const;
77  void cacheMetadataVec(const ChunkMetadataVector&);
78  void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector&, const ChunkKey&) const;
79  bool hasCachedMetadataForKeyPrefix(const ChunkKey&) const;
80  void clearForTablePrefix(const ChunkKey&);
81  void clear();
82  std::vector<ChunkKey> getCachedChunksForKeyPrefix(const ChunkKey&) const;
83  bool recoverCacheForTable(ChunkMetadataVector&, const ChunkKey&);
84  ChunkToBufferMap getChunkBuffersForCaching(
85  const std::vector<ChunkKey>& chunk_keys) const;
86 
87  // Get a chunk buffer for writing to disk prior to metadata creation/caching
88  AbstractBuffer* getChunkBufferForPrecaching(const ChunkKey& chunk_key,
89  bool is_new_buffer);
90 
91  void deleteBufferIfExists(const ChunkKey& chunk_key);
92 
93  inline size_t getNumCachedChunks() const { return cached_chunks_.size(); }
94  inline size_t getNumCachedMetadata() const { return cached_metadata_.size(); }
95  size_t getNumChunksAdded() const { return num_chunks_added_; }
96  size_t getNumMetadataAdded() const { return num_metadata_added_; }
97 
98  // Useful for debugging.
99  std::string dumpCachedChunkEntries() const;
100  std::string dumpCachedMetadataEntries() const;
101 
102  inline std::string getCacheDirectory() const {
103  return caching_file_mgr_->getFileMgrBasePath();
104  }
105 
106  inline std::string getCacheDirectoryForTable(int db_id, int tb_id) const {
107  return caching_file_mgr_->getOrAddTableDir(db_id, tb_id);
108  }
109 
110  void cacheMetadataWithFragIdGreaterOrEqualTo(const ChunkMetadataVector& metadata_vec,
111  const int frag_id);
112  void evictThenEraseChunk(const ChunkKey&);
113 
114  inline uint64_t getSpaceReservedByTable(int db_id, int tb_id) const {
115  return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
116  }
117 
118  private:
119  // These methods are private and assume locks are already acquired when called.
120  std::set<ChunkKey>::iterator eraseChunk(const std::set<ChunkKey>::iterator&);
121  void eraseChunk(const ChunkKey& chunk_key);
122  std::set<ChunkKey>::iterator evictChunkByIterator(
123  const std::set<ChunkKey>::iterator& chunk_it);
124  void evictThenEraseChunkUnlocked(const ChunkKey&);
125  void validatePath(const std::string&) const;
126 
127  // Underlying storage is handled by a CachingFileMgr unique to the cache.
128  std::unique_ptr<File_Namespace::CachingFileMgr> caching_file_mgr_;
129 
130  // Keeps tracks of which Chunks/ChunkMetadata are cached.
131  std::set<ChunkKey> cached_chunks_;
132  std::set<ChunkKey> cached_metadata_;
133 
134  // Keeps tracks of how many times we cache chunks or metadata for testing purposes.
137 
138  // Separate mutexes for chunks/metadata.
141 }; // ForeignStorageCache
142 } // namespace foreign_storage
DiskCacheLevel
DiskCacheLevel enabled_level
std::vector< int > ChunkKey
Definition: types.h:37
bool isEnabledForMutableTables() const
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:58
uint64_t getSpaceReservedByTable(int db_id, int tb_id) const
std::shared_timed_mutex mapd_shared_mutex
bool isEnabledForFSI() const
const std::string wrapper_file_name
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
#define DEFAULT_PAGE_SIZE
CacheTooSmallException(const std::string &msg)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
std::string getCacheDirectoryForTable(int db_id, int tb_id) const
bool isEnabled() const