OmniSciDB  a575cb28ea
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
FileBuffer.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #pragma once
24 
25 #include "DataMgr/AbstractBuffer.h"
26 #include "DataMgr/FileMgr/Page.h"
27 
28 #include <iostream>
29 #include <stdexcept>
30 
31 #include "Logger/Logger.h"
32 
33 using namespace Data_Namespace;
34 
35 #define NUM_METADATA 10
36 #define METADATA_VERSION 0
37 #define METADATA_PAGE_SIZE 4096
38 
39 namespace File_Namespace {
40 
41 class FileMgr; // forward declaration
42 
56 class FileBuffer : public AbstractBuffer {
57  friend class FileMgr;
58 
59  public:
63  FileBuffer(FileMgr* fm,
64  const size_t pageSize,
65  const ChunkKey& chunkKey,
66  const size_t initialSize = 0);
67 
68  FileBuffer(FileMgr* fm,
69  const size_t pageSize,
70  const ChunkKey& chunkKey,
71  const SQLTypeInfo sqlType,
72  const size_t initialSize = 0);
73 
74  FileBuffer(FileMgr* fm,
75  /* const size_t pageSize,*/ const ChunkKey& chunkKey,
76  const std::vector<HeaderInfo>::const_iterator& headerStartIt,
77  const std::vector<HeaderInfo>::const_iterator& headerEndIt);
78 
80  ~FileBuffer() override;
81 
82  Page addNewMultiPage(const int32_t epoch);
83 
84  void reserve(const size_t numBytes) override;
85 
86  void freeMetadataPages();
87  size_t freeChunkPages();
88  void freePages();
89  void freePagesBeforeEpoch(const int32_t targetEpoch);
90 
91  void read(int8_t* const dst,
92  const size_t numBytes = 0,
93  const size_t offset = 0,
94  const MemoryLevel dstMemoryLevel = CPU_LEVEL,
95  const int32_t deviceId = -1) override;
96 
105  void write(int8_t* src,
106  const size_t numBytes,
107  const size_t offset = 0,
108  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
109  const int32_t deviceId = -1) override;
110 
111  void append(int8_t* src,
112  const size_t numBytes,
113  const MemoryLevel srcMemoryLevel = CPU_LEVEL,
114  const int32_t deviceId = -1) override;
115  void copyPage(Page& srcPage,
116  Page& destPage,
117  const size_t numBytes,
118  const size_t offset = 0);
119  inline Data_Namespace::MemoryLevel getType() const override { return DISK_LEVEL; }
120 
122  int8_t* getMemoryPtr() override {
123  LOG(FATAL) << "Operation not supported.";
124  return nullptr; // satisfy return-type warning
125  }
126 
128  inline size_t pageCount() const override { return multiPages_.size(); }
129 
131  inline size_t pageSize() const override { return pageSize_; }
132 
134  inline virtual size_t pageDataSize() const { return pageDataSize_; }
135 
138  inline virtual size_t reservedHeaderSize() const { return reservedHeaderSize_; }
139 
141  inline virtual std::vector<MultiPage> getMultiPage() const { return multiPages_; }
142 
144  inline size_t reservedSize() const override { return multiPages_.size() * pageSize_; }
145 
147  // inline virtual size_t used() const {
148 
149  inline size_t numMetadataPages() const { return metadataPages_.pageVersions.size(); };
150 
151  private:
152  // FileBuffer(const FileBuffer&); // private copy constructor
153  // FileBuffer& operator=(const FileBuffer&); // private overloaded assignment operator
154 
156  // headerSize(numBytes), ChunkKey, pageId, version epoch
157  // void writeHeader(Page &page, const int32_t pageId, const int32_t epoch, const bool
158  // writeSize = false);
159  void writeHeader(Page& page,
160  const int32_t pageId,
161  const int32_t epoch,
162  const bool writeMetadata = false);
163  void writeMetadata(const int32_t epoch);
164  void readMetadata(const Page& page);
165  void calcHeaderBuffer();
166 
167  void freePage(const Page& page, const bool isRolloff);
168  void freePagesBeforeEpochForMultiPage(MultiPage& multiPage,
169  const int32_t targetEpoch,
170  const int32_t currentEpoch);
171 
172  FileMgr* fm_; // a reference to FileMgr is needed for writing to new pages in available
173  // files
174  static size_t headerBufferOffset_;
176  std::vector<MultiPage> multiPages_;
177  size_t pageSize_;
179  size_t reservedHeaderSize_; // lets make this a constant now for simplicity - 128 bytes
181 };
182 
183 } // namespace File_Namespace
virtual std::vector< MultiPage > getMultiPage() const
Returns vector of MultiPages in the FileBuffer.
Definition: FileBuffer.h:141
std::vector< int > ChunkKey
Definition: types.h:37
size_t reservedSize() const override
Returns the total number of bytes allocated for the FileBuffer.
Definition: FileBuffer.h:144
static size_t headerBufferOffset_
Definition: FileBuffer.h:174
A logical page (Page) belongs to a file on disk.
Definition: Page.h:46
#define LOG(tag)
Definition: Logger.h:188
Data_Namespace::MemoryLevel getType() const override
Definition: FileBuffer.h:119
std::vector< MultiPage > multiPages_
Definition: FileBuffer.h:176
Represents/provides access to contiguous data stored in the file system.
Definition: FileBuffer.h:56
size_t read(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Reads the specified number of bytes from the offset position in file f into buf.
Definition: File.cpp:118
size_t pageSize() const override
Returns the size in bytes of each page in the FileBuffer.
Definition: FileBuffer.h:131
An AbstractBuffer is a unit of data management for a data manager.
size_t append(FILE *f, const size_t size, int8_t *buf)
Appends the specified number of bytes to the end of the file f from buf.
Definition: File.cpp:141
size_t pageCount() const override
Returns the number of pages in the FileBuffer.
Definition: FileBuffer.h:128
size_t numMetadataPages() const
Returns the total number of used bytes in the FileBuffer.
Definition: FileBuffer.h:149
virtual size_t reservedHeaderSize() const
Definition: FileBuffer.h:138
size_t write(FILE *f, const size_t offset, const size_t size, int8_t *buf)
Writes the specified number of bytes to the offset position in file f from buf.
Definition: File.cpp:126
int8_t * getMemoryPtr() override
Not implemented for FileMgr – throws a runtime_error.
Definition: FileBuffer.h:122
virtual size_t pageDataSize() const
Returns the size in bytes of the data portion of each page in the FileBuffer.
Definition: FileBuffer.h:134
The MultiPage stores versions of the same logical page in a deque.
Definition: Page.h:72