OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ForeignStorageMgr.h
Go to the documentation of this file.
1 
2 /*
3  * Copyright 2020 OmniSci, Inc.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 #pragma once
19 
20 #include <shared_mutex>
21 
23 #include "ForeignDataWrapper.h"
25 
26 using namespace Data_Namespace;
27 
28 namespace Catalog_Namespace {
29 class Catalog;
30 }
31 
32 class PostEvictionRefreshException : public std::runtime_error {
33  public:
37  PostEvictionRefreshException(const std::runtime_error& exception)
38  : std::runtime_error(""), original_exception_(exception){};
39 
40  std::runtime_error getOriginalException() { return original_exception_; }
41 
42  private:
43  std::runtime_error original_exception_;
44 };
45 
46 namespace foreign_storage {
47 bool is_append_table_chunk_key(const ChunkKey& chunk_key);
48 
50  public:
51  ChunkSizeValidator(const ChunkKey& chunk_key);
52 
53  void validateChunkSize(const AbstractBuffer* buffer) const;
54 
55  void validateChunkSizes(const ChunkToBufferMap& buffers) const;
56 
57  void throwChunkSizeViolatedError(const int64_t actual_chunk_size,
58  const int column_id = -1) const;
59 
60  private:
62  int64_t max_chunk_size_;
63  std::shared_ptr<Catalog_Namespace::Catalog> catalog_;
66 };
67 
68 bool set_comp(const ChunkKey& left, const ChunkKey& right);
69 
70 // For testing purposes only
72  public:
73  virtual void setParentWrapper(
74  std::shared_ptr<ForeignDataWrapper> parent_data_wrapper) = 0;
75 };
76 
78  public:
80 
81  ~ForeignStorageMgr() override {}
82 
83  AbstractBuffer* createBuffer(const ChunkKey& chunk_key,
84  const size_t page_size,
85  const size_t initial_size) override;
86  void deleteBuffer(const ChunkKey& chunk_key, const bool purge) override;
87  void deleteBuffersWithPrefix(const ChunkKey& chunk_key_prefix,
88  const bool purge) override;
89  AbstractBuffer* getBuffer(const ChunkKey& chunk_key, const size_t num_bytes) override;
90  void fetchBuffer(const ChunkKey& chunk_key,
91  AbstractBuffer* destination_buffer,
92  const size_t num_bytes) override;
93  AbstractBuffer* putBuffer(const ChunkKey& chunk_key,
94  AbstractBuffer* source_buffer,
95  const size_t num_bytes) override;
96  /*
97  Obtains chunk-metadata relating to a prefix. Will create and use new
98  datawrappers if none are found for the given prefix.
99  */
100  void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector& chunk_metadata,
101  const ChunkKey& chunk_key_prefix) override;
102  bool isBufferOnDevice(const ChunkKey& chunk_key) override;
103  std::string printSlabs() override;
104  size_t getMaxSize() override;
105  size_t getInUseSize() override;
106  size_t getAllocated() override;
107  bool isAllocationCapped() override;
108  void checkpoint() override;
109  void checkpoint(const int db_id, const int tb_id) override;
110  AbstractBuffer* alloc(const size_t num_bytes) override;
111  void free(AbstractBuffer* buffer) override;
112  MgrType getMgrType() override;
113  std::string getStringMgrType() override;
114  size_t getNumChunks() override;
115  void removeTableRelatedDS(const int db_id, const int table_id) override;
116  bool hasDataWrapperForChunk(const ChunkKey& chunk_key) const;
117  virtual bool createDataWrapperIfNotExists(const ChunkKey& chunk_key);
118 
119  // For testing, is datawrapper state recovered from disk
120  bool isDatawrapperRestored(const ChunkKey& chunk_key);
121  void setDataWrapper(const ChunkKey& table_key,
122  std::shared_ptr<MockForeignDataWrapper> data_wrapper);
123  std::shared_ptr<ForeignDataWrapper> getDataWrapper(const ChunkKey& chunk_key) const;
124 
125  virtual void refreshTable(const ChunkKey& table_key, const bool evict_cached_entries);
126 
127  using ParallelismHint = std::pair<int, int>;
128  void setParallelismHints(
129  const std::map<ChunkKey, std::set<ParallelismHint>>& hints_per_table);
130  virtual size_t maxFetchSize(int32_t db_id) const;
131  virtual bool hasMaxFetchSize() const;
132 
133  protected:
134  virtual void eraseDataWrapper(const ChunkKey& table_key);
135  void updateFragmenterMetadata(const ChunkToBufferMap&) const;
136  void createDataWrapperUnlocked(int32_t db, int32_t tb);
137  bool fetchBufferIfTempBufferMapEntryExists(const ChunkKey& chunk_key,
138  AbstractBuffer* destination_buffer,
139  const size_t num_bytes);
140  ChunkToBufferMap allocateTempBuffersForChunks(const std::set<ChunkKey>& chunk_keys);
141  void clearTempChunkBufferMapEntriesForTable(const ChunkKey& table_key);
142  void clearTempChunkBufferMapEntriesForTableUnlocked(const ChunkKey& table_key);
143 
144  std::set<ChunkKey> getOptionalChunkKeySet(
145  const ChunkKey& chunk_key,
146  const std::set<ChunkKey>& required_chunk_keys,
147  const ForeignDataWrapper::ParallelismLevel parallelism_level) const;
148 
149  std::pair<std::set<ChunkKey, decltype(set_comp)*>,
150  std::set<ChunkKey, decltype(set_comp)*>>
151  getPrefetchSets(const ChunkKey& chunk_key,
152  const std::set<ChunkKey>& required_chunk_keys,
153  const ForeignDataWrapper::ParallelismLevel parallelism_level) const;
154 
155  virtual std::set<ChunkKey> getOptionalKeysWithinSizeLimit(
156  const ChunkKey& chunk_key,
157  const std::set<ChunkKey, decltype(set_comp)*>& same_fragment_keys,
158  const std::set<ChunkKey, decltype(set_comp)*>& diff_fragment_keys) const;
159 
160  static void checkIfS3NeedsToBeEnabled(const ChunkKey& chunk_key);
161 
162  mutable std::shared_mutex data_wrapper_mutex_;
163  std::map<ChunkKey, std::shared_ptr<ForeignDataWrapper>> data_wrapper_map_;
164 
165  // TODO: Remove below map, which is used to temporarily hold chunk buffers,
166  // when buffer mgr interface is updated to accept multiple buffers in one call
167  std::map<ChunkKey, std::unique_ptr<AbstractBuffer>> temp_chunk_buffer_map_;
168  mutable std::shared_mutex temp_chunk_buffer_map_mutex_;
169 
170  mutable std::shared_mutex parallelism_hints_mutex_;
171  std::map<ChunkKey, std::set<ParallelismHint>> parallelism_hints_per_table_;
172 };
173 
174 std::vector<ChunkKey> get_column_key_vec(const ChunkKey& destination_chunk_key);
175 std::set<ChunkKey> get_column_key_set(const ChunkKey& destination_chunk_key);
176 size_t get_max_chunk_size(const ChunkKey& key);
177 bool contains_fragment_key(const std::set<ChunkKey>& key_set, const ChunkKey& target_key);
178 bool is_table_enabled_on_node(const ChunkKey& key);
179 } // namespace foreign_storage
bool set_comp(const ChunkKey &left, const ChunkKey &right)
std::vector< int > ChunkKey
Definition: types.h:37
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
std::shared_ptr< Catalog_Namespace::Catalog > catalog_
bool is_append_table_chunk_key(const ChunkKey &chunk_key)
std::set< ChunkKey > get_column_key_set(const ChunkKey &destination_chunk_key)
const ColumnDescriptor * column_
std::runtime_error getOriginalException()
bool is_table_enabled_on_node(const ChunkKey &key)
PostEvictionRefreshException(const std::runtime_error &exception)
bool contains_fragment_key(const std::set< ChunkKey > &key_set, const ChunkKey &target_key)
std::shared_mutex temp_chunk_buffer_map_mutex_
std::map< ChunkKey, std::set< ParallelismHint > > parallelism_hints_per_table_
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
specifies the content in-memory of a row in the column metadata table
std::map< ChunkKey, std::shared_ptr< ForeignDataWrapper > > data_wrapper_map_
size_t get_max_chunk_size(const ChunkKey &key)
std::runtime_error original_exception_
std::map< ChunkKey, std::unique_ptr< AbstractBuffer > > temp_chunk_buffer_map_
std::vector< ChunkKey > get_column_key_vec(const ChunkKey &destination_chunk_key)