OmniSciDB  b24e664e58
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Buffer.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //
18 // Buffer.cpp
19 // mapd2
20 //
21 // @author Steven Stewart <steve@map-d.com>
22 // @author Todd Mostak <todd@map-d.com>
23 //
24 // Copyright (c) 2014 MapD Technologies, Inc. All rights reserved.
25 //
27 
28 #include <cassert>
29 #include <stdexcept>
30 
32 #include "Shared/Logger.h"
33 
34 namespace Buffer_Namespace {
35 
37  BufferList::iterator seg_it,
38  const int device_id,
39  const size_t page_size,
40  const size_t num_bytes)
41  : AbstractBuffer(device_id)
42  , mem_(0)
43  , bm_(bm)
44  , seg_it_(seg_it)
45  , page_size_(page_size)
46  , num_pages_(0)
47  , pin_count_(0) {
48  pin();
49  // so that the pointer value of this Buffer is stored
50  seg_it_->buffer = this;
51  if (num_bytes > 0) {
52  reserve(num_bytes);
53  }
54 }
55 
57 
58 void Buffer::reserve(const size_t num_bytes) {
59 #ifdef BUFFER_MUTEX
60  boost::unique_lock<boost::shared_mutex> write_lock(read_write_mutex_);
61 #endif
62  size_t num_pages = (num_bytes + page_size_ - 1) / page_size_;
63  // std::cout << "NumPages reserved: " << numPages << std::endl;
64  if (num_pages > num_pages_) {
65  // When running out of cpu buffers, reserveBuffer() will fail and
66  // trigger a SlabTooBig exception, so pageDirtyFlags_ and numPages_
67  // MUST NOT be set until reserveBuffer() returns; otherwise, this
68  // buffer is not properly resized, so any call to FileMgr::fetchBuffer()
69  // will proceed to read(), corrupt heap memory and cause core dump later.
70  seg_it_ = bm_->reserveBuffer(seg_it_, page_size_ * num_pages);
71  page_dirty_flags_.resize(num_pages);
72  num_pages_ = num_pages;
73  }
74 }
75 
76 void Buffer::read(int8_t* const dst,
77  const size_t num_bytes,
78  const size_t offset,
79  const MemoryLevel dst_buffer_type,
80  const int dst_device_id) {
81  if (num_bytes == 0) {
82  return;
83  }
84  CHECK(dst && mem_);
85 #ifdef BUFFER_MUTEX
86  boost::shared_lock<boost::shared_mutex> read_lock(read_write_mutex_);
87 #endif
88 
89  if (num_bytes + offset > size_) {
90  LOG(FATAL) << "Buffer: Out of bounds read error";
91  }
92  readData(dst, num_bytes, offset, dst_buffer_type, dst_device_id);
93 }
94 
95 void Buffer::write(int8_t* src,
96  const size_t num_bytes,
97  const size_t offset,
98  const MemoryLevel src_buffer_type,
99  const int src_device_id) {
100  CHECK_GT(num_bytes, size_t(0)); // cannot write 0 bytes
101 #ifdef BUFFER_MUTEX
102  boost::unique_lock<boost::shared_mutex> write_lock(read_write_mutex_);
103 #endif
104  if (num_bytes + offset > reservedSize()) {
105  reserve(num_bytes + offset);
106  }
107  // write source contents to buffer
108  writeData(src, num_bytes, offset, src_buffer_type, src_device_id);
109 
110  // update dirty flags for buffer and each affected page
111  is_dirty_ = true;
112  if (offset < size_) {
113  is_updated_ = true;
114  }
115  if (offset + num_bytes > size_) {
116  is_appended_ = true;
117  size_ = offset + num_bytes;
118  }
119  // std::cout << "Size after write: " << size_ << std::endl;
120 
121  size_t first_dirty_page = offset / page_size_;
122  size_t last_dirty_page = (offset + num_bytes - 1) / page_size_;
123  for (size_t i = first_dirty_page; i <= last_dirty_page; ++i) {
124  page_dirty_flags_[i] = true;
125  }
126 }
127 
128 void Buffer::append(int8_t* src,
129  const size_t num_bytes,
130  const MemoryLevel src_buffer_type,
131  const int src_device_id) {
132 #ifdef BUFFER_MUTEX
133  boost::shared_lock<boost::shared_mutex> read_lock(
134  read_write_mutex_); // keep another thread from getting a write lock
135  boost::unique_lock<boost::shared_mutex> append_lock(
136  append_mutex_); // keep another thread from getting an append lock
137 #endif
138 
139  is_dirty_ = true;
140  is_appended_ = true;
141 
142  if (num_bytes + size_ > reservedSize()) {
143  reserve(num_bytes + size_);
144  }
145 
146  writeData(src, num_bytes, size_, src_buffer_type, src_device_id);
147  size_ += num_bytes;
148  // Do we worry about dirty flags here or does append avoid them
149 }
150 
152  return mem_;
153 }
154 } // namespace Buffer_Namespace
virtual void writeData(int8_t *const src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int src_device_id=-1)=0
size_t num_pages_
the size of each page in the buffer
Definition: Buffer.h:164
void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int deviceId=-1) override
Definition: Buffer.cpp:128
BufferList::iterator seg_it_
Definition: Buffer.h:162
int8_t * getMemoryPtr() override
Returns a raw, constant (read-only) pointer to the underlying buffer.
Definition: Buffer.cpp:151
~Buffer() override
Destructor.
Definition: Buffer.cpp:56
#define LOG(tag)
Definition: Logger.h:185
int64_t * src
#define CHECK_GT(x, y)
Definition: Logger.h:202
Note(s): Forbid Copying Idiom 4.1.
Definition: BufferMgr.h:97
CHECK(cgen_state)
BufferList::iterator reserveBuffer(BufferList::iterator &seg_it, const size_t num_bytes)
Definition: BufferMgr.cpp:182
std::vector< bool > page_dirty_flags_
indicates when the buffer was last flushed
Definition: Buffer.h:166
An AbstractBuffer is a unit of data management for a data manager.
virtual void readData(int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0
void write(int8_t *src, const size_t num_bytes, const size_t offset=0, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1) override
Writes (copies) data from src into the buffer. Writes (copies) nbytes of data into the buffer at the ...
Definition: Buffer.cpp:95
void read(int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int device_id=-1) override
Reads (copies) data from the buffer to the destination (dst) memory location. Reads (copies) nbytes o...
Definition: Buffer.cpp:76
Buffer(BufferMgr *bm, BufferList::iterator seg_it, const int device_id, const size_t page_size=512, const size_t num_bytes=0)
Constructs a Buffer object. The constructor requires a memory address (provided by BufferMgr)...
Definition: Buffer.cpp:36
size_t reservedSize() const override
Returns the total number of bytes allocated for the buffer.
Definition: Buffer.h:119
void reserve(const size_t num_bytes) override
Definition: Buffer.cpp:58
int pin() override
Definition: Buffer.h:130