OmniSciDB  8a228a1076
FileBuffer.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #pragma once
24 
25 #include "DataMgr/AbstractBuffer.h"
26 #include "DataMgr/FileMgr/Page.h"
27 
28 #include <iostream>
29 #include <stdexcept>
30 
31 #include "Logger/Logger.h"
32 
33 using namespace Data_Namespace;
34 
35 #define NUM_METADATA 10
36 #define METADATA_VERSION 0
37 
38 namespace File_Namespace {
39 
40 class FileMgr; // forward declaration
41 
55 class FileBuffer : public AbstractBuffer {
56  friend class FileMgr;
57 
58  public:
62  FileBuffer(FileMgr* fm,
63  const size_t pageSize,
64  const ChunkKey& chunkKey,
65  const size_t initialSize = 0);
66 
67  FileBuffer(FileMgr* fm,
68  const size_t pageSize,
69  const ChunkKey& chunkKey,
70  const SQLTypeInfo sqlType,
71  const size_t initialSize = 0);
72 
73  FileBuffer(FileMgr* fm,
74  /* const size_t pageSize,*/ const ChunkKey& chunkKey,
75  const std::vector<HeaderInfo>::const_iterator& headerStartIt,
76  const std::vector<HeaderInfo>::const_iterator& headerEndIt);
77 
79  ~FileBuffer() override;
80 
81  Page addNewMultiPage(const int epoch);
82 
83  void reserve(const size_t numBytes) override;
84 
85  void freePages();
86  void freeChunkPages();
87  void freeMetadataPages();
88 
89  void read(int8_t* const dst,
90  const size_t numBytes = 0,
91  const size_t offset = 0,
92  const MemoryLevel dstMemoryLevel = CPU_LEVEL,
93  const int deviceId = -1) override;
94 
103  void write(int8_t* src,
104  const size_t numBytes,
105  const size_t offset = 0,
106  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
107  const int deviceId = -1) override;
108 
109  void append(int8_t* src,
110  const size_t numBytes,
111  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
112  const int deviceId = -1) override;
113  void copyPage(Page& srcPage,
114  Page& destPage,
115  const size_t numBytes,
116  const size_t offset = 0);
117  inline Data_Namespace::MemoryLevel getType() const override { return DISK_LEVEL; }
118 
120  int8_t* getMemoryPtr() override {
121  LOG(FATAL) << "Operation not supported.";
122  return nullptr; // satisfy return-type warning
123  }
124 
126  inline size_t pageCount() const override { return multiPages_.size(); }
127 
129  inline size_t pageSize() const override { return pageSize_; }
130 
132  inline virtual size_t pageDataSize() const { return pageDataSize_; }
133 
136  inline virtual size_t reservedHeaderSize() const { return reservedHeaderSize_; }
137 
139  inline virtual std::vector<MultiPage> getMultiPage() const { return multiPages_; }
140 
141  inline size_t size() const override { return size_; }
142 
144  inline size_t reservedSize() const override { return multiPages_.size() * pageSize_; }
145 
147  // inline virtual size_t used() const {
148 
151  bool isDirty() const override { return is_dirty_; }
152 
153  private:
154  // FileBuffer(const FileBuffer&); // private copy constructor
155  // FileBuffer& operator=(const FileBuffer&); // private overloaded assignment operator
156 
158  // headerSize(numBytes), ChunkKey, pageId, version epoch
159  // void writeHeader(Page &page, const int pageId, const int epoch, const bool writeSize
160  // = false);
161  void writeHeader(Page& page,
162  const int pageId,
163  const int epoch,
164  const bool writeMetadata = false);
165  void writeMetadata(const int epoch);
166  void readMetadata(const Page& page);
167  void calcHeaderBuffer();
168 
169  FileMgr* fm_; // a reference to FileMgr is needed for writing to new pages in available
170  // files
171  static size_t headerBufferOffset_;
173  std::vector<MultiPage> multiPages_;
174  size_t pageSize_;
176  size_t reservedHeaderSize_; // lets make this a constant now for simplicity - 128 bytes
178 };
179 
180 } // namespace File_Namespace
int64_t * src
virtual size_t pageDataSize() const
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition: FileBuffer.h:132
size_t reservedSize() const override
Returns the total number of bytes allocated for the FileBuffer.
Definition: FileBuffer.h:144
size_t size() const override
Definition: FileBuffer.h:141
static size_t headerBufferOffset_
Definition: FileBuffer.h:171
bool isDirty() const override
Returns the total number of used bytes in the FileBuffer.
Definition: FileBuffer.h:151
A logical page (Page) belongs to a file on disk.
Definition: Page.h:46
#define LOG(tag)
Definition: Logger.h:188
Data_Namespace::MemoryLevel getType() const override
Definition: FileBuffer.h:117
std::vector< MultiPage > multiPages_
Definition: FileBuffer.h:173
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:55
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:117
size_t pageSize() const override
Returns the size in bytes of each page in the FileBuffer.
Definition: FileBuffer.h:129
An AbstractBuffer is a unit of data management for a data manager.
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:140
size_t pageCount() const override
Returns the number of pages in the FileBuffer.
Definition: FileBuffer.h:126
virtual std::vector< MultiPage > getMultiPage() const
Returns vector of MultiPages in the FileBuffer.
Definition: FileBuffer.h:139
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:125
std::vector< int > ChunkKey
Definition: types.h:35
virtual size_t reservedHeaderSize() const
Definition: FileBuffer.h:136
int8_t * getMemoryPtr() override
Not implemented for FileMgr – throws a runtime_error.
Definition: FileBuffer.h:120
The MultiPage stores versions of the same logical page in a deque.
Definition: Page.h:69