OmniSciDB  c07336695a
FileBuffer.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #ifndef DATAMGR_MEMORY_FILE_FILEBUFFER_H
24 #define DATAMGR_MEMORY_FILE_FILEBUFFER_H
25 
26 #include "../AbstractBuffer.h"
27 #include "Page.h"
28 #include "Shared/Logger.h"
29 
30 #include <iostream>
31 #include <stdexcept>
32 
33 using namespace Data_Namespace;
34 
35 #define NUM_METADATA 10
36 #define METADATA_VERSION 0
37 
38 namespace File_Namespace {
39 
40 class FileMgr; // forward declaration
41 
55 class FileBuffer : public AbstractBuffer {
56  friend class FileMgr;
57 
58  public:
62  FileBuffer(FileMgr* fm,
63  const size_t pageSize,
64  const ChunkKey& chunkKey,
65  const size_t initialSize = 0);
66 
67  FileBuffer(FileMgr* fm,
68  const size_t pageSize,
69  const ChunkKey& chunkKey,
70  const SQLTypeInfo sqlType,
71  const size_t initialSize = 0);
72 
73  FileBuffer(FileMgr* fm,
74  /* const size_t pageSize,*/ const ChunkKey& chunkKey,
75  const std::vector<HeaderInfo>::const_iterator& headerStartIt,
76  const std::vector<HeaderInfo>::const_iterator& headerEndIt);
77 
79  ~FileBuffer() override;
80 
81  Page addNewMultiPage(const int epoch);
82 
83  void reserve(const size_t numBytes) override;
84 
85  void freePages();
86 
87  void read(int8_t* const dst,
88  const size_t numBytes = 0,
89  const size_t offset = 0,
90  const MemoryLevel dstMemoryLevel = CPU_LEVEL,
91  const int deviceId = -1) override;
92 
101  void write(int8_t* src,
102  const size_t numBytes,
103  const size_t offset = 0,
104  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
105  const int deviceId = -1) override;
106 
107  void append(int8_t* src,
108  const size_t numBytes,
109  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
110  const int deviceId = -1) override;
111  void copyPage(Page& srcPage,
112  Page& destPage,
113  const size_t numBytes,
114  const size_t offset = 0);
115  inline Data_Namespace::MemoryLevel getType() const override { return DISK_LEVEL; }
116 
118  int8_t* getMemoryPtr() override {
119  LOG(FATAL) << "Operation not supported.";
120  return nullptr; // satisfy return-type warning
121  }
122 
124  inline size_t pageCount() const override { return multiPages_.size(); }
125 
127  inline size_t pageSize() const override { return pageSize_; }
128 
130  inline virtual size_t pageDataSize() const { return pageDataSize_; }
131 
134  inline virtual size_t reservedHeaderSize() const { return reservedHeaderSize_; }
135 
137  inline virtual std::vector<MultiPage> getMultiPage() const { return multiPages_; }
138 
139  inline size_t size() const override { return size_; }
140 
142  inline size_t reservedSize() const override { return multiPages_.size() * pageSize_; }
143 
145  // inline virtual size_t used() const {
146 
149  bool isDirty() const override { return isDirty_; }
150 
151  private:
152  // FileBuffer(const FileBuffer&); // private copy constructor
153  // FileBuffer& operator=(const FileBuffer&); // private overloaded assignment operator
154 
156  // headerSize(numBytes), ChunkKey, pageId, version epoch
157  // void writeHeader(Page &page, const int pageId, const int epoch, const bool writeSize
158  // = false);
159  void writeHeader(Page& page,
160  const int pageId,
161  const int epoch,
162  const bool writeMetadata = false);
163  void writeMetadata(const int epoch);
164  void readMetadata(const Page& page);
165  void calcHeaderBuffer();
166 
167  FileMgr* fm_; // a reference to FileMgr is needed for writing to new pages in available
168  // files
169  static size_t headerBufferOffset_;
171  std::vector<MultiPage> multiPages_;
172  size_t pageSize_;
174  size_t reservedHeaderSize_; // lets make this a constant now for simplicity - 128 bytes
176 };
177 
178 } // namespace File_Namespace
179 
180 #endif // DATAMGR_MEMORY_FILE_FILEBUFFER_H
virtual size_t pageDataSize() const
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition: FileBuffer.h:130
size_t reservedSize() const override
Returns the total number of bytes allocated for the FileBuffer.
Definition: FileBuffer.h:142
size_t size() const override
Definition: FileBuffer.h:139
static size_t headerBufferOffset_
Definition: FileBuffer.h:169
bool isDirty() const override
Returns the total number of used bytes in the FileBuffer.
Definition: FileBuffer.h:149
A logical page (Page) belongs to a file on disk.
Definition: Page.h:46
#define LOG(tag)
Definition: Logger.h:182
Data_Namespace::MemoryLevel getType() const override
Definition: FileBuffer.h:115
int64_t * src
std::vector< MultiPage > multiPages_
Definition: FileBuffer.h:171
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:113
size_t pageSize() const override
Returns the size in bytes of each page in the FileBuffer.
Definition: FileBuffer.h:127
An AbstractBuffer is a unit of data management for a data manager.
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:136
size_t pageCount() const override
Returns the number of pages in the FileBuffer.
Definition: FileBuffer.h:124
virtual std::vector< MultiPage > getMultiPage() const
Returns vector of MultiPages in the FileBuffer.
Definition: FileBuffer.h:137
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:121
std::vector< int > ChunkKey
Definition: types.h:35
virtual size_t reservedHeaderSize() const
Definition: FileBuffer.h:134
int8_t * getMemoryPtr() override
Not implemented for FileMgr – throws a runtime_error.
Definition: FileBuffer.h:118
The MultiPage stores versions of the same logical page in a deque.
Definition: Page.h:69