OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ForeignStorageCache.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
26 #pragma once
27 
28 #include "../Shared/heavyai_shared_mutex.h"
31 #include "ForeignDataWrapper.h"
32 
33 class CacheTooSmallException : public std::runtime_error {
34  public:
35  CacheTooSmallException(const std::string& msg) : std::runtime_error(msg) {}
36 };
37 
38 using namespace Data_Namespace;
39 
40 namespace foreign_storage {
41 
43  public:
45 
46  void checkpoint(const int32_t db_id, const int32_t tb_id);
47  void putBuffer(const ChunkKey&, AbstractBuffer*, const size_t numBytes = 0);
48  File_Namespace::FileBuffer* getCachedChunkIfExists(const ChunkKey&);
49  bool isMetadataCached(const ChunkKey&) const;
50  void cacheMetadataVec(const ChunkMetadataVector&);
51  void getCachedMetadataVecForKeyPrefix(ChunkMetadataVector&, const ChunkKey&) const;
52  bool hasCachedMetadataForKeyPrefix(const ChunkKey&) const;
53  void clearForTablePrefix(const ChunkKey&);
54  void clear();
55  size_t getMaxChunkDataSize() const { return caching_file_mgr_->getMaxDataFilesSize(); }
56  std::vector<ChunkKey> getCachedChunksForKeyPrefix(const ChunkKey&) const;
57 
58  ChunkToBufferMap getChunkBuffersForCaching(const std::set<ChunkKey>& chunk_keys) const;
59 
60  // Get a chunk buffer for writing to disk prior to metadata creation/caching
61  AbstractBuffer* getChunkBufferForPrecaching(const ChunkKey& chunk_key,
62  bool is_new_buffer);
63 
64  void deleteBufferIfExists(const ChunkKey& chunk_key);
65 
66  inline size_t getNumCachedChunks() const {
67  return caching_file_mgr_->getNumDataChunks();
68  }
69  inline size_t getNumCachedMetadata() const {
70  return caching_file_mgr_->getNumChunksWithMetadata();
71  }
72 
73  // Useful for debugging.
74  std::string dumpCachedChunkEntries() const;
75  std::string dumpCachedMetadataEntries() const;
76  std::string dumpEvictionQueue() const;
77  std::string dump() const { return caching_file_mgr_->dump(); }
78 
79  inline std::string getCacheDirectory() const {
80  return caching_file_mgr_->getFileMgrBasePath();
81  }
82 
83  inline std::string getCacheDirectoryForTable(int db_id, int tb_id) const {
84  return caching_file_mgr_->getTableFileMgrPath(db_id, tb_id);
85  }
86 
87  inline std::string getSerializedWrapperPath(int32_t db_id, int32_t tb_id) const {
88  return getCacheDirectoryForTable(db_id, tb_id) + "/" +
90  }
91 
92  inline uint64_t getSpaceReservedByTable(int db_id, int tb_id) const {
93  return caching_file_mgr_->getSpaceReservedByTable(db_id, tb_id);
94  }
95 
96  void storeDataWrapper(const std::string& doc, int32_t db_id, int32_t tb_id);
97 
98  bool hasStoredDataWrapperMetadata(int32_t db_id, int32_t table_id) const;
99 
100  // Used for unit testing
101  inline void setDataSizeLimit(size_t max) const {
102  caching_file_mgr_->setDataSizeLimit(max);
103  }
104 
105  private:
106  // These methods are private and assume locks are already acquired when called.
107  std::set<ChunkKey>::iterator eraseChunk(const std::set<ChunkKey>::iterator&);
108  void eraseChunk(const ChunkKey& chunk_key);
109  void validatePath(const std::string&) const;
110 
111  // Underlying storage is handled by a CachingFileMgr unique to the cache.
112  std::unique_ptr<File_Namespace::CachingFileMgr> caching_file_mgr_;
113 
114 }; // ForeignStorageCache
115 } // namespace foreign_storage
std::vector< int > ChunkKey
Definition: types.h:36
This file details an extension of the FileMgr that can contain pages from multiple tables (CachingFil...
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:58
uint64_t getSpaceReservedByTable(int db_id, int tb_id) const
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
std::string getSerializedWrapperPath(int32_t db_id, int32_t tb_id) const
CacheTooSmallException(const std::string &msg)
std::unique_ptr< File_Namespace::CachingFileMgr > caching_file_mgr_
std::string getCacheDirectoryForTable(int db_id, int tb_id) const
static constexpr char WRAPPER_FILE_NAME[]