OmniSciDB  bf83d84833
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ForeignStorageCache.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #pragma once
27 
28 #include <gtest/gtest.h>
29 #include "../Shared/mapd_shared_mutex.h"
32 #include "ForeignDataWrapper.h"
33 
34 class CacheTooSmallException : public std::runtime_error {
35  public:
36  CacheTooSmallException(const std::string& msg) : std::runtime_error(msg) {}
37 };
38 
39 enum class DiskCacheLevel { none, fsi, non_fsi, all };
41  std::string path;
43  size_t num_reader_threads = 0;
44  inline bool isEnabledForMutableTables() const {
47  }
48  inline bool isEnabledForFSI() const {
50  }
51  inline bool isEnabled() const { return enabled_level != DiskCacheLevel::none; }
52 };
53 
54 using namespace Data_Namespace;
55 
56 namespace foreign_storage {
57 
59  public:
60  ForeignStorageCache(const DiskCacheConfig& config);
61 
70  void cacheTableChunks(const std::vector<ChunkKey>& chunk_keys);
71  void cacheChunk(const ChunkKey&, AbstractBuffer*);
72 
73  AbstractBuffer* getCachedChunkIfExists(const ChunkKey&);
74  bool isMetadataCached(const ChunkKey&) const;
75  void cacheMetadataVec(const ChunkMetadataVector&);
76  void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector&, const ChunkKey&) const;
77  bool hasCachedMetadataForKeyPrefix(const ChunkKey&) const;
78  void clearForTablePrefix(const ChunkKey&);
79  void clear();
80  std::vector<ChunkKey> getCachedChunksForKeyPrefix(const ChunkKey&) const;
81  bool recoverCacheForTable(ChunkMetadataVector&, const ChunkKey&);
82  std::map<ChunkKey, AbstractBuffer*> getChunkBuffersForCaching(
83  const std::vector<ChunkKey>& chunk_keys) const;
84 
85  // Get a chunk buffer for writing to disk prior to metadata creation/caching
86  AbstractBuffer* getChunkBufferForPrecaching(const ChunkKey& chunk_key,
87  bool is_new_buffer);
88 
89  void deleteBufferIfExists(const ChunkKey& chunk_key);
90 
91  inline size_t getNumCachedChunks() const { return cached_chunks_.size(); }
92  inline size_t getNumCachedMetadata() const { return cached_metadata_.size(); }
93  size_t getNumChunksAdded() const { return num_chunks_added_; }
94  size_t getNumMetadataAdded() const { return num_metadata_added_; }
95 
96  // Useful for debugging.
97  std::string dumpCachedChunkEntries() const;
98  std::string dumpCachedMetadataEntries() const;
99 
101  return global_file_mgr_.get();
102  }
103 
104  std::string getCacheDirectoryForTablePrefix(const ChunkKey&) const;
105  void cacheMetadataWithFragIdGreaterOrEqualTo(const ChunkMetadataVector& metadata_vec,
106  const int frag_id);
107  void evictThenEraseChunk(const ChunkKey&);
108 
109  private:
110  // These methods are private and assume locks are already acquired when called.
111  std::set<ChunkKey>::iterator eraseChunk(const std::set<ChunkKey>::iterator&);
112  void eraseChunk(const ChunkKey& chunk_key);
113  std::set<ChunkKey>::iterator evictChunkByIterator(
114  const std::set<ChunkKey>::iterator& chunk_it);
115  void evictThenEraseChunkUnlocked(const ChunkKey&);
116  void validatePath(const std::string&) const;
117 
118  // Underlying storage is handled by a GlobalFileMgr unique to the cache.
119  std::unique_ptr<File_Namespace::GlobalFileMgr> global_file_mgr_;
120 
121  // Keeps tracks of which Chunks/ChunkMetadata are cached.
122  std::set<ChunkKey> cached_chunks_;
123  std::set<ChunkKey> cached_metadata_;
124 
125  // Keeps tracks of how many times we cache chunks or metadata for testing purposes.
128 
129  // Separate mutexes for chunks/metadata.
132 }; // ForeignStorageCache
133 } // namespace foreign_storage
DiskCacheLevel
DiskCacheLevel enabled_level
std::vector< int > ChunkKey
Definition: types.h:37
bool isEnabledForMutableTables() const
std::shared_timed_mutex mapd_shared_mutex
bool isEnabledForFSI() const
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
CacheTooSmallException(const std::string &msg)
std::unique_ptr< File_Namespace::GlobalFileMgr > global_file_mgr_
bool isEnabled() const