OmniSciDB  c0231cc57d
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
FileBuffer.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #pragma once
24 
25 #include "DataMgr/AbstractBuffer.h"
26 #include "DataMgr/FileMgr/Page.h"
27 
28 #include <iostream>
29 #include <stdexcept>
30 
31 #include "Logger/Logger.h"
32 
33 using namespace Data_Namespace;
34 
35 #define NUM_METADATA 10
36 #define METADATA_VERSION 0
37 
38 namespace File_Namespace {
39 
40 // forward declarations
41 class FileMgr;
42 class CachingFileMgr;
43 
57 class FileBuffer : public AbstractBuffer {
58  friend class FileMgr;
59  friend class CachingFileMgr;
60 
61  public:
65  FileBuffer(FileMgr* fm,
66  const size_t pageSize,
67  const ChunkKey& chunkKey,
68  const size_t initialSize = 0);
69 
70  FileBuffer(FileMgr* fm,
71  const size_t pageSize,
72  const ChunkKey& chunkKey,
73  const SQLTypeInfo sqlType,
74  const size_t initialSize = 0);
75 
76  FileBuffer(FileMgr* fm,
77  /* const size_t pageSize,*/ const ChunkKey& chunkKey,
78  const std::vector<HeaderInfo>::const_iterator& headerStartIt,
79  const std::vector<HeaderInfo>::const_iterator& headerEndIt);
80 
82  ~FileBuffer() override;
83 
84  Page addNewMultiPage(const int32_t epoch);
85 
86  void reserve(const size_t numBytes) override;
87 
88  size_t freeMetadataPages();
89  size_t freeChunkPages();
90  size_t freePages();
91  void freePagesBeforeEpoch(const int32_t targetEpoch);
92 
93  void read(int8_t* const dst,
94  const size_t numBytes = 0,
95  const size_t offset = 0,
96  const MemoryLevel dstMemoryLevel = CPU_LEVEL,
97  const int32_t deviceId = -1) override;
98 
107  void write(int8_t* src,
108  const size_t numBytes,
109  const size_t offset = 0,
110  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
111  const int32_t deviceId = -1) override;
112 
113  void append(int8_t* src,
114  const size_t numBytes,
115  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
116  const int32_t deviceId = -1) override;
117  void copyPage(Page& srcPage,
118  Page& destPage,
119  const size_t numBytes,
120  const size_t offset = 0);
121  inline Data_Namespace::MemoryLevel getType() const override { return DISK_LEVEL; }
122 
124  int8_t* getMemoryPtr() override {
125  LOG(FATAL) << "Operation not supported.";
126  return nullptr; // satisfy return-type warning
127  }
128 
130  inline size_t pageCount() const override { return multiPages_.size(); }
131 
134  inline bool hasDataPages() const { return pageCount() > 0; }
135 
137  inline size_t pageSize() const override { return pageSize_; }
138 
140  inline virtual size_t pageDataSize() const { return pageDataSize_; }
141 
144  inline virtual size_t reservedHeaderSize() const { return reservedHeaderSize_; }
145 
147  inline virtual std::vector<MultiPage> getMultiPage() const { return multiPages_; }
148  inline MultiPage getMetadataPage() const { return metadataPages_; }
149 
151  inline size_t reservedSize() const override { return multiPages_.size() * pageSize_; }
152 
154  // inline virtual size_t used() const {
155 
156  inline size_t numMetadataPages() const { return metadataPages_.pageVersions.size(); };
157 
158  bool isMissingPages() const;
159  size_t numChunkPages() const;
160  std::string dump() const;
161 
162  // Used for testing
163  void freePage(const Page& page);
164 
165  static constexpr size_t headerBufferOffset_ = 32;
166 
167  private:
168  // FileBuffer(const FileBuffer&); // private copy constructor
169  // FileBuffer& operator=(const FileBuffer&); // private overloaded assignment operator
170 
172  // headerSize(numBytes), ChunkKey, pageId, version epoch
173  // void writeHeader(Page &page, const int32_t pageId, const int32_t epoch, const bool
174  // writeSize = false);
175  void writeHeader(Page& page,
176  const int32_t pageId,
177  const int32_t epoch,
178  const bool writeMetadata = false);
179  void writeMetadata(const int32_t epoch);
180  void readMetadata(const Page& page);
181  void calcHeaderBuffer();
182 
183  void freePage(const Page& page, const bool isRolloff);
184  void freePagesBeforeEpochForMultiPage(MultiPage& multiPage,
185  const int32_t targetEpoch,
186  const int32_t currentEpoch);
187  void initMetadataAndPageDataSize();
188  int32_t getFileMgrEpoch();
189 
190  FileMgr* fm_; // a reference to FileMgr is needed for writing to new pages in available
191  // files
192  // pageSize_ is non-const because it can be read from a metadata file to create a
193  // non-standard page size. metadataPageSize_ is const because we need to know what the
194  // metadata page size is in order to know which files are metadata files (and therefore
195  // determine page size). This is set earlier in FileMgr construction.
196  const size_t metadataPageSize_;
198  std::vector<MultiPage> multiPages_;
199  size_t pageSize_;
201  size_t reservedHeaderSize_; // lets make this a constant now for simplicity - 128 bytes
203 };
204 
205 } // namespace File_Namespace
virtual std::vector< MultiPage > getMultiPage() const
Returns vector of MultiPages in the FileBuffer.
Definition: FileBuffer.h:147
std::vector< int > ChunkKey
Definition: types.h:36
size_t append(FILE *f, const size_t size, const int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:168
size_t reservedSize() const override
Returns the total number of bytes allocated for the FileBuffer.
Definition: FileBuffer.h:151
A logical page (Page) belongs to a file on disk.
Definition: Page.h:46
#define LOG(tag)
Definition: Logger.h:216
Data_Namespace::MemoryLevel getType() const override
Definition: FileBuffer.h:121
const size_t metadataPageSize_
Definition: FileBuffer.h:196
std::vector< MultiPage > multiPages_
Definition: FileBuffer.h:198
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:57
size_t write(FILE *f, const size_t offset, const size_t size, const int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:150
MultiPage getMetadataPage() const
Definition: FileBuffer.h:148
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:142
size_t pageSize() const override
Returns the size in bytes of each page in the FileBuffer.
Definition: FileBuffer.h:137
An AbstractBuffer is a unit of data management for a data manager.
size_t pageCount() const override
Returns the number of pages in the FileBuffer.
Definition: FileBuffer.h:130
size_t numMetadataPages() const
Returns the total number of used bytes in the FileBuffer.
Definition: FileBuffer.h:156
virtual size_t reservedHeaderSize() const
Definition: FileBuffer.h:144
bool hasDataPages() const
Definition: FileBuffer.h:134
This file contains the declaration and definition of a Page type and a MultiPage type.
int8_t * getMemoryPtr() override
Not implemented for FileMgr – throws a runtime_error.
Definition: FileBuffer.h:124
virtual size_t pageDataSize() const
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition: FileBuffer.h:140
The MultiPage stores versions of the same logical page in a deque.
Definition: Page.h:79
A FileMgr capable of limiting it&#39;s size and storing data from multiple tables in a shared directory...