OmniSciDB  c0231cc57d
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ForeignStorageInterface.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include "Catalog/CatalogFwd.h"
21 #include "LockMgr/LockMgr.h"
22 
23 #include <atomic>
24 #include <unordered_map>
25 
29  const std::vector<int8_t> buff;
30 };
31 
33  public:
35 
36  virtual void append(const std::vector<ForeignStorageColumnBuffer>& column_buffers) = 0;
37  virtual void read(const ChunkKey& chunk_key,
38  const SQLTypeInfo& sql_type,
39  int8_t* dest,
40  const size_t num_bytes) = 0;
41  virtual int8_t* tryZeroCopy(const ChunkKey& chunk_key,
42  const SQLTypeInfo& sql_type,
43  const size_t num_bytes) {
44  return nullptr;
45  }
46  virtual void prepareTable(const int /*db_id*/,
47  const std::string& type,
48  TableDescriptor& /*td*/,
49  std::list<ColumnDescriptor>& /*cols*/) {}
50  virtual void registerTable(Catalog_Namespace::Catalog* catalog,
51  std::pair<int, int> table_key,
52  const std::string& type,
53  const TableDescriptor& td,
54  const std::list<ColumnDescriptor>& cols,
55  Data_Namespace::AbstractBufferMgr* mgr) = 0;
56  virtual void dropTable(const int db_id, const int table_id) = 0;
57  virtual std::string getType() const = 0;
58 };
59 
61  public:
62  ForeignStorageBuffer(const ChunkKey& chunk_key,
63  PersistentForeignStorageInterface* persistent_foreign_storage);
64 
65  void read(int8_t* const dst,
66  const size_t numBytes,
67  const size_t offset = 0,
69  const int dstDeviceId = -1) override;
70 
71  void append(int8_t* src,
72  const size_t numBytes,
74  const int deviceId = -1) override;
75 
78  };
79 
80  std::vector<int8_t> moveBuffer() { return std::move(buff_); }
81 
82  void write(int8_t* src,
83  const size_t numBytes,
84  const size_t offset = 0,
86  const int srcDeviceId = -1) override {
87  CHECK(false);
88  }
89 
90  void reserve(size_t numBytes) override { CHECK(false); }
91 
92  int8_t* getMemoryPtr() override {
93  CHECK(false);
94  return nullptr;
95  }
96 
97  size_t pageCount() const override {
98  CHECK(false);
99  return 0;
100  }
101 
102  size_t pageSize() const override {
103  CHECK(false);
104  return 0;
105  }
106 
107  size_t reservedSize() const override {
108  CHECK(false);
109  return 0;
110  }
111 
112  int8_t* tryZeroCopy(const size_t numBytes);
113 
114  private:
117  std::vector<int8_t> buff_;
118 };
119 
120 class ForeignStorageBufferMgr : public Data_Namespace::AbstractBufferMgr {
121  public:
122  ForeignStorageBufferMgr(const int db_id,
123  const int table_id,
124  PersistentForeignStorageInterface* persistent_foreign_storage);
125 
126  void checkpoint() override;
127 
129  const size_t pageSize = 0,
130  const size_t initialSize = 0) override;
131 
133  const size_t numBytes = 0) override;
134 
135  void fetchBuffer(const ChunkKey& key,
136  Data_Namespace::AbstractBuffer* destBuffer,
137  const size_t numBytes = 0) override;
138 
140  const ChunkKey& keyPrefix) override;
141  std::string getStringMgrType() override { return ToString(FILE_MGR); }
142 
143  size_t getNumChunks() override {
144  heavyai::shared_lock<heavyai::shared_mutex> chunk_index_write_lock(
146  return chunk_index_.size();
147  }
148 
149  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
150  CHECK(false);
151  }
152 
153  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
154  const bool purge = true) override {
155  CHECK(false);
156  }
157 
160  const size_t numBytes = 0) override {
161  CHECK(false);
162  return nullptr;
163  }
164 
165  bool isBufferOnDevice(const ChunkKey& key) override {
166  CHECK(false);
167  return false;
168  }
169 
170  std::string printSlabs() override {
171  CHECK(false);
172  return "";
173  }
174 
175  size_t getMaxSize() override {
176  CHECK(false);
177  return 0;
178  }
179 
180  size_t getInUseSize() override {
181  CHECK(false);
182  return 0;
183  }
184 
185  size_t getAllocated() override {
186  CHECK(false);
187  return 0;
188  }
189 
190  bool isAllocationCapped() override {
191  CHECK(false);
192  return false;
193  }
194 
195  void checkpoint(const int db_id, const int tb_id) override { CHECK(false); }
196 
197  // Buffer API
198  Data_Namespace::AbstractBuffer* alloc(const size_t numBytes = 0) override {
199  CHECK(false);
200  return nullptr;
201  }
202 
203  void free(Data_Namespace::AbstractBuffer* buffer) override { CHECK(false); }
204 
205  MgrType getMgrType() override {
206  CHECK(false);
207  return FILE_MGR;
208  }
209 
210  void removeTableRelatedDS(const int db_id, const int table_id) override {
211  persistent_foreign_storage_->dropTable(db_id, table_id);
212  }
213 
214  private:
216  std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer>> chunk_index_;
218 };
219 
221  public:
224 
225  ForeignStorageInterface(const ForeignStorageInterface& other) = delete;
227 
230 
231  Data_Namespace::AbstractBufferMgr* lookupBufferManager(const int db_id,
232  const int table_id);
233  void dropBufferManager(const int db_id, const int table_id);
234 
236  std::unique_ptr<PersistentForeignStorageInterface> persistent_foreign_storage);
237 
239  void prepareTable(const int db_id,
240  TableDescriptor& td,
241  std::list<ColumnDescriptor>& cols);
244  const TableDescriptor& td,
245  const std::list<ColumnDescriptor>& cols);
246 
247  private:
248  std::unordered_map<std::string, std::unique_ptr<PersistentForeignStorageInterface>>
250  std::map<std::pair<int, int>, PersistentForeignStorageInterface*>
252  std::map<std::pair<int, int>, std::unique_ptr<ForeignStorageBufferMgr>> managers_map_;
254 };
void reserve(size_t numBytes) override
PersistentForeignStorageInterface * persistent_foreign_storage_
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
std::vector< int > ChunkKey
Definition: types.h:36
void checkpoint(const int db_id, const int tb_id) override
std::unordered_map< std::string, std::unique_ptr< PersistentForeignStorageInterface > > persistent_storage_interfaces_
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:132
virtual void registerTable(Catalog_Namespace::Catalog *catalog, std::pair< int, int > table_key, const std::string &type, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols, Data_Namespace::AbstractBufferMgr *mgr)=0
Data_Namespace::AbstractBufferMgr * lookupBufferManager(const int db_id, const int table_id)
void append(int8_t *src, const size_t numBytes, const Data_Namespace::MemoryLevel srcBufferType=Data_Namespace::CPU_LEVEL, const int deviceId=-1) override
ForeignStorageInterface & operator=(const ForeignStorageInterface &other)=delete
size_t reservedSize() const override
std::string getStringMgrType() override
std::vector< int8_t > moveBuffer()
size_t pageSize() const override
void write(int8_t *src, const size_t numBytes, const size_t offset=0, const Data_Namespace::MemoryLevel srcBufferType=Data_Namespace::CPU_LEVEL, const int srcDeviceId=-1) override
void free(Data_Namespace::AbstractBuffer *buffer) override
void fetchBuffer(const ChunkKey &key, Data_Namespace::AbstractBuffer *destBuffer, const size_t numBytes=0) override
int8_t * getMemoryPtr() override
std::shared_lock< T > shared_lock
void registerPersistentStorageInterface(std::unique_ptr< PersistentForeignStorageInterface > persistent_foreign_storage)
Data_Namespace::AbstractBuffer * putBuffer(const ChunkKey &key, Data_Namespace::AbstractBuffer *srcBuffer, const size_t numBytes=0) override
bool isBufferOnDevice(const ChunkKey &key) override
ForeignStorageBufferMgr(const int db_id, const int table_id, PersistentForeignStorageInterface *persistent_foreign_storage)
virtual void prepareTable(const int, const std::string &type, TableDescriptor &, std::list< ColumnDescriptor > &)
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
virtual void append(const std::vector< ForeignStorageColumnBuffer > &column_buffers)=0
ForeignStorageBuffer(const ChunkKey &chunk_key, PersistentForeignStorageInterface *persistent_foreign_storage)
An AbstractBuffer is a unit of data management for a data manager.
void registerTable(Catalog_Namespace::Catalog *catalog, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols)
ids are created
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
std::vector< int8_t > buff_
virtual void dropTable(const int db_id, const int table_id)=0
Data_Namespace::AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
virtual int8_t * tryZeroCopy(const ChunkKey &chunk_key, const SQLTypeInfo &sql_type, const size_t num_bytes)
Data_Namespace::AbstractBuffer * alloc(const size_t numBytes=0) override
std::map< std::pair< int, int >, PersistentForeignStorageInterface * > table_persistent_storage_interface_map_
heavyai::shared_mutex chunk_index_mutex_
void dropBufferManager(const int db_id, const int table_id)
#define CHECK(condition)
Definition: Logger.h:222
Data_Namespace::MemoryLevel getType() const override
std::map< std::pair< int, int >, std::unique_ptr< ForeignStorageBufferMgr > > managers_map_
std::string printSlabs() override
PersistentForeignStorageInterface * persistent_foreign_storage_
std::shared_timed_mutex shared_mutex
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
void read(int8_t *const dst, const size_t numBytes, const size_t offset=0, const Data_Namespace::MemoryLevel dstBufferType=Data_Namespace::CPU_LEVEL, const int dstDeviceId=-1) override
size_t pageCount() const override
virtual std::string getType() const =0
const std::vector< int8_t > buff
virtual void read(const ChunkKey &chunk_key, const SQLTypeInfo &sql_type, int8_t *dest, const size_t num_bytes)=0
int8_t * tryZeroCopy(const size_t numBytes)
void removeTableRelatedDS(const int db_id, const int table_id) override
void prepareTable(const int db_id, TableDescriptor &td, std::list< ColumnDescriptor > &cols)
prepare table options and modify columns
Data_Namespace::AbstractBuffer * createBuffer(const ChunkKey &key, const size_t pageSize=0, const size_t initialSize=0) override