OmniSciDB  3a86f6ec37
ForeignStorageInterface.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include "../AbstractBufferMgr.h"
20 #include "Catalog/Catalog.h"
21 
22 #include <unordered_map>
23 
27  const std::vector<int8_t> buff;
28 };
29 
31  public:
33 
34  virtual void append(const std::vector<ForeignStorageColumnBuffer>& column_buffers) = 0;
35  virtual void read(const ChunkKey& chunk_key,
36  const SQLTypeInfo& sql_type,
37  int8_t* dest,
38  const size_t num_bytes) = 0;
39  virtual int8_t* tryZeroCopy(const ChunkKey& chunk_key,
40  const SQLTypeInfo& sql_type,
41  const size_t num_bytes) {
42  return nullptr;
43  }
44  virtual void prepareTable(const int /*db_id*/,
45  const std::string& type,
46  TableDescriptor& /*td*/,
47  std::list<ColumnDescriptor>& /*cols*/) {}
48  virtual void registerTable(Catalog_Namespace::Catalog* catalog,
49  std::pair<int, int> table_key,
50  const std::string& type,
51  const TableDescriptor& td,
52  const std::list<ColumnDescriptor>& cols,
53  Data_Namespace::AbstractBufferMgr* mgr) = 0;
54  virtual std::string getType() const = 0;
55 };
56 
58  public:
60  PersistentForeignStorageInterface* persistent_foreign_storage);
61 
62  void read(int8_t* const dst,
63  const size_t numBytes,
64  const size_t offset = 0,
66  const int dstDeviceId = -1) override;
67 
68  void append(int8_t* src,
69  const size_t numBytes,
71  const int deviceId = -1) override;
72 
75  };
76 
77  std::vector<int8_t> moveBuffer() { return std::move(buff_); }
78 
79  void write(int8_t* src,
80  const size_t numBytes,
81  const size_t offset = 0,
83  const int srcDeviceId = -1) override {
84  CHECK(false);
85  }
86 
87  void reserve(size_t numBytes) override { CHECK(false); }
88 
89  int8_t* getMemoryPtr() override {
90  CHECK(false);
91  return nullptr;
92  }
93 
94  size_t pageCount() const override {
95  CHECK(false);
96  return 0;
97  }
98 
99  size_t pageSize() const override {
100  CHECK(false);
101  return 0;
102  }
103 
104  size_t reservedSize() const override {
105  CHECK(false);
106  return 0;
107  }
108 
109  int8_t* tryZeroCopy(const size_t numBytes);
110 
111  private:
114  std::vector<int8_t> buff_;
115 };
116 
117 class ForeignStorageBufferMgr : public Data_Namespace::AbstractBufferMgr {
118  public:
119  ForeignStorageBufferMgr(const int db_id,
120  const int table_id,
121  PersistentForeignStorageInterface* persistent_foreign_storage);
122 
123  void checkpoint() override;
124 
125  Data_Namespace::AbstractBuffer* createBuffer(const ChunkKey& key,
126  const size_t pageSize = 0,
127  const size_t initialSize = 0) override;
128 
129  Data_Namespace::AbstractBuffer* getBuffer(const ChunkKey& key,
130  const size_t numBytes = 0) override;
131 
132  void fetchBuffer(const ChunkKey& key,
133  Data_Namespace::AbstractBuffer* destBuffer,
134  const size_t numBytes = 0) override;
135 
136  void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector& chunkMetadataVec,
137  const ChunkKey& keyPrefix) override;
138  std::string getStringMgrType() override { return ToString(FILE_MGR); }
139 
140  size_t getNumChunks() override {
141  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
142  return chunk_index_.size();
143  }
144 
145  void deleteBuffer(const ChunkKey& key, const bool purge = true) override {
146  CHECK(false);
147  }
148 
149  void deleteBuffersWithPrefix(const ChunkKey& keyPrefix,
150  const bool purge = true) override {
151  CHECK(false);
152  }
153 
156  const size_t numBytes = 0) override {
157  CHECK(false);
158  return nullptr;
159  }
160 
161  bool isBufferOnDevice(const ChunkKey& key) override {
162  CHECK(false);
163  return false;
164  }
165 
166  std::string printSlabs() override {
167  CHECK(false);
168  return "";
169  }
170 
171  void clearSlabs() override { CHECK(false); }
172 
173  size_t getMaxSize() override {
174  CHECK(false);
175  return 0;
176  }
177 
178  size_t getInUseSize() override {
179  CHECK(false);
180  return 0;
181  }
182 
183  size_t getAllocated() override {
184  CHECK(false);
185  return 0;
186  }
187 
188  bool isAllocationCapped() override {
189  CHECK(false);
190  return false;
191  }
192 
193  void checkpoint(const int db_id, const int tb_id) override { CHECK(false); }
194 
195  // Buffer API
196  Data_Namespace::AbstractBuffer* alloc(const size_t numBytes = 0) override {
197  CHECK(false);
198  return nullptr;
199  }
200 
201  void free(Data_Namespace::AbstractBuffer* buffer) override { CHECK(false); }
202 
203  MgrType getMgrType() override {
204  CHECK(false);
205  return FILE_MGR;
206  }
207 
208  void removeTableRelatedDS(const int db_id, const int table_id) override {
209  UNREACHABLE();
210  }
211 
212  private:
214  std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer>> chunk_index_;
216 };
217 
219  public:
220  static Data_Namespace::AbstractBufferMgr* lookupBufferManager(const int db_id,
221  const int table_id);
222 
223  static void registerPersistentStorageInterface(
224  std::unique_ptr<PersistentForeignStorageInterface> persistent_foreign_storage);
225 
226  static void destroy();
227 
229  static void prepareTable(const int db_id,
230  TableDescriptor& td,
231  std::list<ColumnDescriptor>& cols);
233  static void registerTable(Catalog_Namespace::Catalog* catalog,
234  const TableDescriptor& td,
235  const std::list<ColumnDescriptor>& cols);
236 
237  private:
238  static std::unordered_map<std::string,
239  std::unique_ptr<PersistentForeignStorageInterface>>
241  static std::map<std::pair<int, int>, PersistentForeignStorageInterface*>
243  static std::map<std::pair<int, int>, std::unique_ptr<ForeignStorageBufferMgr>>
246 };
void reserve(size_t numBytes) override
PersistentForeignStorageInterface * persistent_foreign_storage_
void deleteBuffer(const ChunkKey &key, const bool purge=true) override
void checkpoint(const int db_id, const int tb_id) override
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:101
size_t reservedSize() const override
#define UNREACHABLE()
Definition: Logger.h:241
std::string getStringMgrType() override
std::vector< int8_t > moveBuffer()
size_t pageSize() const override
void write(int8_t *src, const size_t numBytes, const size_t offset=0, const Data_Namespace::MemoryLevel srcBufferType=Data_Namespace::CPU_LEVEL, const int srcDeviceId=-1) override
void free(Data_Namespace::AbstractBuffer *buffer) override
int8_t * getMemoryPtr() override
This file contains the class specification and related data structures for Catalog.
Data_Namespace::AbstractBuffer * putBuffer(const ChunkKey &key, Data_Namespace::AbstractBuffer *srcBuffer, const size_t numBytes=0) override
bool isBufferOnDevice(const ChunkKey &key) override
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:118
std::shared_timed_mutex mapd_shared_mutex
virtual void prepareTable(const int, const std::string &type, TableDescriptor &, std::list< ColumnDescriptor > &)
mapd_shared_mutex chunk_index_mutex_
An AbstractBuffer is a unit of data management for a data manager.
static std::map< std::pair< int, int >, std::unique_ptr< ForeignStorageBufferMgr > > managers_map_
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:141
static std::mutex persistent_storage_interfaces_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
std::vector< int8_t > buff_
virtual int8_t * tryZeroCopy(const ChunkKey &chunk_key, const SQLTypeInfo &sql_type, const size_t num_bytes)
Data_Namespace::AbstractBuffer * alloc(const size_t numBytes=0) override
static std::map< std::pair< int, int >, PersistentForeignStorageInterface * > table_persistent_storage_interface_map_
#define CHECK(condition)
Definition: Logger.h:197
Data_Namespace::MemoryLevel getType() const override
std::vector< int > ChunkKey
Definition: types.h:37
static std::unordered_map< std::string, std::unique_ptr< PersistentForeignStorageInterface > > persistent_storage_interfaces_
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata > >> ChunkMetadataVector
std::string printSlabs() override
PersistentForeignStorageInterface * persistent_foreign_storage_
void deleteBuffersWithPrefix(const ChunkKey &keyPrefix, const bool purge=true) override
size_t pageCount() const override
const std::vector< int8_t > buff
void removeTableRelatedDS(const int db_id, const int table_id) override