OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
FileBuffer.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #pragma once
24 
25 #include "DataMgr/AbstractBuffer.h"
26 #include "DataMgr/FileMgr/Page.h"
27 
28 #include <iostream>
29 #include <stdexcept>
30 
31 #include "Logger/Logger.h"
32 
33 using namespace Data_Namespace;
34 
35 #define NUM_METADATA 10
36 #define METADATA_VERSION 0
37 #define METADATA_PAGE_SIZE 4096
38 
39 namespace File_Namespace {
40 
41 // forward declarations
42 class FileMgr;
43 class CachingFileMgr;
44 
58 class FileBuffer : public AbstractBuffer {
59  friend class FileMgr;
60  friend class CachingFileMgr;
61 
62  public:
66  FileBuffer(FileMgr* fm,
67  const size_t pageSize,
68  const ChunkKey& chunkKey,
69  const size_t initialSize = 0);
70 
71  FileBuffer(FileMgr* fm,
72  const size_t pageSize,
73  const ChunkKey& chunkKey,
74  const SQLTypeInfo sqlType,
75  const size_t initialSize = 0);
76 
77  FileBuffer(FileMgr* fm,
78  /* const size_t pageSize,*/ const ChunkKey& chunkKey,
79  const std::vector<HeaderInfo>::const_iterator& headerStartIt,
80  const std::vector<HeaderInfo>::const_iterator& headerEndIt);
81 
83  virtual ~FileBuffer() override;
84 
85  Page addNewMultiPage(const int32_t epoch);
86 
87  void reserve(const size_t numBytes) override;
88 
89  void freeMetadataPages();
90  size_t freeChunkPages();
91  void freePages();
92  void freePagesBeforeEpoch(const int32_t targetEpoch);
93 
94  void read(int8_t* const dst,
95  const size_t numBytes = 0,
96  const size_t offset = 0,
97  const MemoryLevel dstMemoryLevel = CPU_LEVEL,
98  const int32_t deviceId = -1) override;
99 
108  void write(int8_t* src,
109  const size_t numBytes,
110  const size_t offset = 0,
111  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
112  const int32_t deviceId = -1) override;
113 
114  void append(int8_t* src,
115  const size_t numBytes,
116  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
117  const int32_t deviceId = -1) override;
118  void copyPage(Page& srcPage,
119  Page& destPage,
120  const size_t numBytes,
121  const size_t offset = 0);
122  inline Data_Namespace::MemoryLevel getType() const override { return DISK_LEVEL; }
123 
125  int8_t* getMemoryPtr() override {
126  LOG(FATAL) << "Operation not supported.";
127  return nullptr; // satisfy return-type warning
128  }
129 
131  inline size_t pageCount() const override { return multiPages_.size(); }
132 
134  inline size_t pageSize() const override { return pageSize_; }
135 
137  inline virtual size_t pageDataSize() const { return pageDataSize_; }
138 
141  inline virtual size_t reservedHeaderSize() const { return reservedHeaderSize_; }
142 
144  inline virtual std::vector<MultiPage> getMultiPage() const { return multiPages_; }
145  inline MultiPage getMetadataPage() const { return metadataPages_; }
146 
148  inline size_t reservedSize() const override { return multiPages_.size() * pageSize_; }
149 
151  // inline virtual size_t used() const {
152 
153  inline size_t numMetadataPages() const { return metadataPages_.pageVersions.size(); };
154  inline size_t numChunkPages() const {
155  size_t total_size = 0;
156  for (const auto& multi_page : multiPages_) {
157  total_size += multi_page.pageVersions.size();
158  }
159  return total_size;
160  }
161 
162  std::string dump() const;
163 
164  bool isMissingPages() const;
165 
166  // Used for testing
167  void freePage(const Page& page);
168 
169  static constexpr size_t headerBufferOffset_ = 32;
170 
171  private:
172  // FileBuffer(const FileBuffer&); // private copy constructor
173  // FileBuffer& operator=(const FileBuffer&); // private overloaded assignment operator
174 
176  // headerSize(numBytes), ChunkKey, pageId, version epoch
177  // void writeHeader(Page &page, const int32_t pageId, const int32_t epoch, const bool
178  // writeSize = false);
179  void writeHeader(Page& page,
180  const int32_t pageId,
181  const int32_t epoch,
182  const bool writeMetadata = false);
183  void writeMetadata(const int32_t epoch);
184  void readMetadata(const Page& page);
185  void calcHeaderBuffer();
186 
187  void freePage(const Page& page, const bool isRolloff);
188  void freePagesBeforeEpochForMultiPage(MultiPage& multiPage,
189  const int32_t targetEpoch,
190  const int32_t currentEpoch);
191  void initMetadataAndPageDataSize();
192  int32_t getFileMgrEpoch();
193 
194  FileMgr* fm_; // a reference to FileMgr is needed for writing to new pages in available
195  // files
197  std::vector<MultiPage> multiPages_;
198  size_t pageSize_;
200  size_t reservedHeaderSize_; // lets make this a constant now for simplicity - 128 bytes
202 };
203 
204 } // namespace File_Namespace
virtual std::vector< MultiPage > getMultiPage() const
Returns vector of MultiPages in the FileBuffer.
Definition: FileBuffer.h:144
std::vector< int > ChunkKey
Definition: types.h:37
size_t append(FILE *f, const size_t size, const int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:159
size_t reservedSize() const override
Returns the total number of bytes allocated for the FileBuffer.
Definition: FileBuffer.h:148
A logical page (Page) belongs to a file on disk.
Definition: Page.h:46
#define LOG(tag)
Definition: Logger.h:194
size_t numChunkPages() const
Definition: FileBuffer.h:154
Data_Namespace::MemoryLevel getType() const override
Definition: FileBuffer.h:122
std::vector< MultiPage > multiPages_
Definition: FileBuffer.h:197
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:58
size_t write(FILE *f, const size_t offset, const size_t size, const int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:141
MultiPage getMetadataPage() const
Definition: FileBuffer.h:145
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:133
size_t pageSize() const override
Returns the size in bytes of each page in the FileBuffer.
Definition: FileBuffer.h:134
An AbstractBuffer is a unit of data management for a data manager.
size_t pageCount() const override
Returns the number of pages in the FileBuffer.
Definition: FileBuffer.h:131
size_t numMetadataPages() const
Returns the total number of used bytes in the FileBuffer.
Definition: FileBuffer.h:153
virtual size_t reservedHeaderSize() const
Definition: FileBuffer.h:141
int8_t * getMemoryPtr() override
Not implemented for FileMgr – throws a runtime_error.
Definition: FileBuffer.h:125
virtual size_t pageDataSize() const
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition: FileBuffer.h:137
The MultiPage stores versions of the same logical page in a deque.
Definition: Page.h:79
A FileMgr capable of limiting it&#39;s size and storing data from multiple tables in a shared directory...