OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ForeignStorageMgr.h
Go to the documentation of this file.
1 
2 /*
3  * Copyright 2020 OmniSci, Inc.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #pragma once
19 
20 #include <shared_mutex>
21 
23 #include "ForeignDataWrapper.h"
25 
26 using namespace Data_Namespace;
27 
28 class PostEvictionRefreshException : public std::runtime_error {
29  public:
33  PostEvictionRefreshException(const std::runtime_error& exception)
34  : std::runtime_error(""), original_exception_(exception){};
35 
36  std::runtime_error getOriginalException() { return original_exception_; }
37 
38  private:
39  std::runtime_error original_exception_;
40 };
41 
42 namespace foreign_storage {
43 
44 // For testing purposes only
46  public:
47  virtual void setParentWrapper(
48  std::shared_ptr<ForeignDataWrapper> parent_data_wrapper) = 0;
49 };
50 
52  public:
54 
55  ~ForeignStorageMgr() override {}
56 
57  AbstractBuffer* createBuffer(const ChunkKey& chunk_key,
58  const size_t page_size,
59  const size_t initial_size) override;
60  void deleteBuffer(const ChunkKey& chunk_key, const bool purge) override;
61  void deleteBuffersWithPrefix(const ChunkKey& chunk_key_prefix,
62  const bool purge) override;
63  AbstractBuffer* getBuffer(const ChunkKey& chunk_key, const size_t num_bytes) override;
64  void fetchBuffer(const ChunkKey& chunk_key,
65  AbstractBuffer* destination_buffer,
66  const size_t num_bytes) override;
67  AbstractBuffer* putBuffer(const ChunkKey& chunk_key,
68  AbstractBuffer* source_buffer,
69  const size_t num_bytes) override;
70  /*
71  Obtains chunk-metadata relating to a prefix. Will create and use new
72  datawrappers if none are found for the given prefix.
73  */
74  void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector& chunk_metadata,
75  const ChunkKey& chunk_key_prefix) override;
76  bool isBufferOnDevice(const ChunkKey& chunk_key) override;
77  std::string printSlabs() override;
78  size_t getMaxSize() override;
79  size_t getInUseSize() override;
80  size_t getAllocated() override;
81  bool isAllocationCapped() override;
82  void checkpoint() override;
83  void checkpoint(const int db_id, const int tb_id) override;
84  AbstractBuffer* alloc(const size_t num_bytes) override;
85  void free(AbstractBuffer* buffer) override;
86  MgrType getMgrType() override;
87  std::string getStringMgrType() override;
88  size_t getNumChunks() override;
89  void removeTableRelatedDS(const int db_id, const int table_id) override;
90  bool hasDataWrapperForChunk(const ChunkKey& chunk_key);
91  virtual bool createDataWrapperIfNotExists(const ChunkKey& chunk_key);
92 
93  // For testing, is datawrapper state recovered from disk
94  bool isDatawrapperRestored(const ChunkKey& chunk_key);
95  void setDataWrapper(const ChunkKey& table_key,
96  std::shared_ptr<MockForeignDataWrapper> data_wrapper);
97 
98  virtual void refreshTable(const ChunkKey& table_key, const bool evict_cached_entries);
99 
100  using ParallelismHint = std::pair<int, int>;
101  void setParallelismHints(
102  const std::map<ChunkKey, std::set<ParallelismHint>>& hints_per_table);
103 
104  protected:
105  void createDataWrapperUnlocked(int32_t db, int32_t tb);
106  void clearDataWrapper(const ChunkKey& table_key);
107  std::shared_ptr<ForeignDataWrapper> getDataWrapper(const ChunkKey& chunk_key);
108  bool fetchBufferIfTempBufferMapEntryExists(const ChunkKey& chunk_key,
109  AbstractBuffer* destination_buffer,
110  const size_t num_bytes);
111  ChunkToBufferMap allocateTempBuffersForChunks(const std::set<ChunkKey>& chunk_keys);
112  void clearTempChunkBufferMapEntriesForTable(const ChunkKey& table_key);
113  void clearTempChunkBufferMapEntriesForTableUnlocked(const ChunkKey& table_key);
114 
115  void getOptionalChunkKeySet(
116  std::set<ChunkKey>& optional_chunk_keys,
117  const ChunkKey& chunk_key,
118  const std::set<ChunkKey>& required_chunk_keys,
119  const ForeignDataWrapper::ParallelismLevel parallelism_level);
120 
121  static void checkIfS3NeedsToBeEnabled(const ChunkKey& chunk_key);
122 
123  std::shared_mutex data_wrapper_mutex_;
124  std::map<ChunkKey, std::shared_ptr<ForeignDataWrapper>> data_wrapper_map_;
125 
126  // TODO: Remove below map, which is used to temporarily hold chunk buffers,
127  // when buffer mgr interface is updated to accept multiple buffers in one call
128  std::map<ChunkKey, std::unique_ptr<AbstractBuffer>> temp_chunk_buffer_map_;
129  std::shared_mutex temp_chunk_buffer_map_mutex_;
130 
131  std::shared_mutex parallelism_hints_mutex_;
132  std::map<ChunkKey, std::set<ParallelismHint>> parallelism_hints_per_table_;
133 };
134 
135 std::vector<ChunkKey> get_keys_vec_from_table(const ChunkKey& destination_chunk_key);
136 std::set<ChunkKey> get_keys_set_from_table(const ChunkKey& destination_chunk_key);
137 } // namespace foreign_storage
std::vector< ChunkKey > get_keys_vec_from_table(const ChunkKey &destination_chunk_key)
std::vector< int > ChunkKey
Definition: types.h:37
std::set< ChunkKey > get_keys_set_from_table(const ChunkKey &destination_chunk_key)
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
std::runtime_error getOriginalException()
PostEvictionRefreshException(const std::runtime_error &exception)
std::shared_mutex temp_chunk_buffer_map_mutex_
std::map< ChunkKey, std::set< ParallelismHint > > parallelism_hints_per_table_
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
std::map< ChunkKey, std::shared_ptr< ForeignDataWrapper > > data_wrapper_map_
std::runtime_error original_exception_
std::map< ChunkKey, std::unique_ptr< AbstractBuffer > > temp_chunk_buffer_map_