OmniSciDB  94e8789169
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ForeignStorageBufferMgr Class Reference

#include <ForeignStorageInterface.h>

+ Inheritance diagram for ForeignStorageBufferMgr:
+ Collaboration diagram for ForeignStorageBufferMgr:

Public Member Functions

 ForeignStorageBufferMgr (const int db_id, const int table_id, PersistentForeignStorageInterface *persistent_foreign_storage)
 
void checkpoint () override
 
Data_Namespace::AbstractBuffercreateBuffer (const ChunkKey &key, const size_t pageSize=0, const size_t initialSize=0) override
 
Data_Namespace::AbstractBuffergetBuffer (const ChunkKey &key, const size_t numBytes=0) override
 
void fetchBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *destBuffer, const size_t numBytes=0) override
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
 
std::string getStringMgrType () override
 
size_t getNumChunks () override
 
void deleteBuffer (const ChunkKey &key, const bool purge=true) override
 
void deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
 
Data_Namespace::AbstractBufferputBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *srcBuffer, const size_t numBytes=0) override
 
bool isBufferOnDevice (const ChunkKey &key) override
 
std::string printSlabs () override
 
void clearSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void checkpoint (const int db_id, const int tb_id) override
 
Data_Namespace::AbstractBufferalloc (const size_t numBytes=0) override
 
void free (Data_Namespace::AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 
void removeTableRelatedDS (const int db_id, const int table_id) override
 

Private Attributes

PersistentForeignStorageInterfacepersistent_foreign_storage_
 
std::map< ChunkKey,
std::unique_ptr
< ForeignStorageBuffer > > 
chunk_index_
 
mapd_shared_mutex chunk_index_mutex_
 

Detailed Description

Definition at line 117 of file ForeignStorageInterface.h.

Constructor & Destructor Documentation

ForeignStorageBufferMgr::ForeignStorageBufferMgr ( const int  db_id,
const int  table_id,
PersistentForeignStorageInterface persistent_foreign_storage 
)

Definition at line 50 of file ForeignStorageInterface.cpp.

54  : AbstractBufferMgr(0), persistent_foreign_storage_(persistent_foreign_storage) {}
PersistentForeignStorageInterface * persistent_foreign_storage_

Member Function Documentation

Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::alloc ( const size_t  numBytes = 0)
inlineoverride

Definition at line 196 of file ForeignStorageInterface.h.

References CHECK.

196  {
197  CHECK(false);
198  return nullptr;
199  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::checkpoint ( )
override

Definition at line 56 of file ForeignStorageInterface.cpp.

References PersistentForeignStorageInterface::append(), chunk_index_, and persistent_foreign_storage_.

56  {
57  // TODO(alex)
58  std::vector<ForeignStorageColumnBuffer> column_buffers;
59  for (auto& kv : chunk_index_) {
60  const auto buffer = kv.second->moveBuffer();
61  column_buffers.emplace_back(
62  ForeignStorageColumnBuffer{kv.first, kv.second->getSqlType(), buffer});
63  }
64  persistent_foreign_storage_->append(column_buffers);
65 }
PersistentForeignStorageInterface * persistent_foreign_storage_
virtual void append(const std::vector< ForeignStorageColumnBuffer > &column_buffers)=0
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::checkpoint ( const int  db_id,
const int  tb_id 
)
inlineoverride

Definition at line 193 of file ForeignStorageInterface.h.

References CHECK.

193 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::clearSlabs ( )
inlineoverride

Definition at line 171 of file ForeignStorageInterface.h.

References CHECK.

171 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:197
Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::createBuffer ( const ChunkKey key,
const size_t  pageSize = 0,
const size_t  initialSize = 0 
)
override

Definition at line 67 of file ForeignStorageInterface.cpp.

References chunk_index_, chunk_index_mutex_, and persistent_foreign_storage_.

70  {
71  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
72  const auto it_ok = chunk_index_.emplace(
73  key, std::make_unique<ForeignStorageBuffer>(key, persistent_foreign_storage_));
74  // this check fails if we create table, drop it and create again
75  // CHECK(it_ok.second);
76  return it_ok.first->second.get();
77 }
PersistentForeignStorageInterface * persistent_foreign_storage_
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
void ForeignStorageBufferMgr::deleteBuffer ( const ChunkKey key,
const bool  purge = true 
)
inlineoverride

Definition at line 145 of file ForeignStorageInterface.h.

References CHECK.

145  {
146  CHECK(false);
147  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::deleteBuffersWithPrefix ( const ChunkKey keyPrefix,
const bool  purge = true 
)
inlineoverride

Definition at line 149 of file ForeignStorageInterface.h.

References CHECK.

150  {
151  CHECK(false);
152  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::fetchBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer destBuffer,
const size_t  numBytes = 0 
)
override

Definition at line 88 of file ForeignStorageInterface.cpp.

References CHECK, getBuffer(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::setMemoryPtr(), Data_Namespace::AbstractBuffer::setSize(), and Data_Namespace::AbstractBuffer::syncEncoder().

90  {
91  CHECK(numBytes);
92  auto file_buffer = dynamic_cast<ForeignStorageBuffer*>(getBuffer(key, numBytes));
93  CHECK(file_buffer);
94 
95  // TODO: check if GPU is used
96  auto buf = file_buffer->tryZeroCopy(numBytes);
97  if (buf) {
98  destBuffer->setMemoryPtr(buf);
99  } else {
100  destBuffer->reserve(numBytes);
101  file_buffer->read(destBuffer->getMemoryPtr(), numBytes);
102  }
103  destBuffer->setSize(numBytes);
104  destBuffer->syncEncoder(file_buffer);
105 }
void syncEncoder(const AbstractBuffer *src_buffer)
virtual int8_t * getMemoryPtr()=0
virtual void setMemoryPtr(int8_t *new_ptr)
Data_Namespace::AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
void setSize(const size_t size)
#define CHECK(condition)
Definition: Logger.h:197
virtual void reserve(size_t num_bytes)=0

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::free ( Data_Namespace::AbstractBuffer buffer)
inlineoverride

Definition at line 201 of file ForeignStorageInterface.h.

References CHECK.

201 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getAllocated ( )
inlineoverride

Definition at line 183 of file ForeignStorageInterface.h.

References CHECK.

183  {
184  CHECK(false);
185  return 0;
186  }
#define CHECK(condition)
Definition: Logger.h:197
Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::getBuffer ( const ChunkKey key,
const size_t  numBytes = 0 
)
override

Definition at line 79 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, and chunk_index_mutex_.

Referenced by fetchBuffer().

81  {
82  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
83  const auto it = chunk_index_.find(key);
84  CHECK(it != chunk_index_.end());
85  return it->second.get();
86 }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the caller graph for this function:

void ForeignStorageBufferMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunkMetadataVec,
const ChunkKey keyPrefix 
)
override

Definition at line 107 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, chunk_index_mutex_, and run_benchmark_import::type.

109  {
110  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(
111  chunk_index_mutex_); // is this guarding the right structure? it look slike we
112  // oly read here for chunk
113  auto chunk_it = chunk_index_.lower_bound(keyPrefix);
114  if (chunk_it == chunk_index_.end()) {
115  CHECK(false); // throw?
116  }
117 
118  while (chunk_it != chunk_index_.end() &&
119  std::search(chunk_it->first.begin(),
120  chunk_it->first.begin() + keyPrefix.size(),
121  keyPrefix.begin(),
122  keyPrefix.end()) != chunk_it->first.begin() + keyPrefix.size()) {
123  const auto& chunk_key = chunk_it->first;
124  if (chunk_key.size() == 5) {
125  if (chunk_key[4] == 1) {
126  const auto& buffer = *chunk_it->second;
127  auto type = buffer.getSqlType();
128  auto size = buffer.size();
129  auto subkey = chunk_key;
130  subkey[4] = 2;
131  auto& index_buf = *(chunk_index_.find(subkey)->second);
132  auto bs = index_buf.size() / index_buf.getSqlType().get_size();
133  auto chunk_metadata =
134  std::make_shared<ChunkMetadata>(type, size, bs, ChunkStats{});
135  chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
136  }
137  } else {
138  const auto& buffer = *chunk_it->second;
139  auto chunk_metadata = std::make_shared<ChunkMetadata>();
140  chunk_metadata->sqlType = buffer.getSqlType();
141  buffer.getEncoder()->getMetadata(chunk_metadata);
142  chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
143  }
144  chunk_it++;
145  }
146 }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getInUseSize ( )
inlineoverride

Definition at line 178 of file ForeignStorageInterface.h.

References CHECK.

178  {
179  CHECK(false);
180  return 0;
181  }
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getMaxSize ( )
inlineoverride

Definition at line 173 of file ForeignStorageInterface.h.

References CHECK.

173  {
174  CHECK(false);
175  return 0;
176  }
#define CHECK(condition)
Definition: Logger.h:197
MgrType ForeignStorageBufferMgr::getMgrType ( )
inlineoverride

Definition at line 203 of file ForeignStorageInterface.h.

References CHECK.

203  {
204  CHECK(false);
205  return FILE_MGR;
206  }
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getNumChunks ( )
inlineoverride

Definition at line 140 of file ForeignStorageInterface.h.

References chunk_index_, and chunk_index_mutex_.

140  {
141  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
142  return chunk_index_.size();
143  }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
std::string ForeignStorageBufferMgr::getStringMgrType ( )
inlineoverride

Definition at line 138 of file ForeignStorageInterface.h.

138 { return ToString(FILE_MGR); }
bool ForeignStorageBufferMgr::isAllocationCapped ( )
inlineoverride

Definition at line 188 of file ForeignStorageInterface.h.

References CHECK.

188  {
189  CHECK(false);
190  return false;
191  }
#define CHECK(condition)
Definition: Logger.h:197
bool ForeignStorageBufferMgr::isBufferOnDevice ( const ChunkKey key)
inlineoverride

Definition at line 161 of file ForeignStorageInterface.h.

References CHECK.

161  {
162  CHECK(false);
163  return false;
164  }
#define CHECK(condition)
Definition: Logger.h:197
std::string ForeignStorageBufferMgr::printSlabs ( )
inlineoverride

Definition at line 166 of file ForeignStorageInterface.h.

References CHECK.

166  {
167  CHECK(false);
168  return "";
169  }
#define CHECK(condition)
Definition: Logger.h:197
Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::putBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer srcBuffer,
const size_t  numBytes = 0 
)
inlineoverride

Definition at line 154 of file ForeignStorageInterface.h.

References CHECK.

156  {
157  CHECK(false);
158  return nullptr;
159  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::removeTableRelatedDS ( const int  db_id,
const int  table_id 
)
inlineoverride

Definition at line 208 of file ForeignStorageInterface.h.

References UNREACHABLE.

208  {
209  UNREACHABLE();
210  }
#define UNREACHABLE()
Definition: Logger.h:241

Member Data Documentation

std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer> > ForeignStorageBufferMgr::chunk_index_
private
mapd_shared_mutex ForeignStorageBufferMgr::chunk_index_mutex_
mutableprivate
PersistentForeignStorageInterface* ForeignStorageBufferMgr::persistent_foreign_storage_
private

Definition at line 213 of file ForeignStorageInterface.h.

Referenced by checkpoint(), and createBuffer().


The documentation for this class was generated from the following files: