OmniSciDB  d2f719934e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ForeignStorageBufferMgr Class Reference

#include <ForeignStorageInterface.h>

+ Inheritance diagram for ForeignStorageBufferMgr:
+ Collaboration diagram for ForeignStorageBufferMgr:

Public Member Functions

 ForeignStorageBufferMgr (const int db_id, const int table_id, PersistentForeignStorageInterface *persistent_foreign_storage)
 
void checkpoint () override
 
Data_Namespace::AbstractBuffercreateBuffer (const ChunkKey &key, const size_t pageSize=0, const size_t initialSize=0) override
 
Data_Namespace::AbstractBuffergetBuffer (const ChunkKey &key, const size_t numBytes=0) override
 
void fetchBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *destBuffer, const size_t numBytes=0) override
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
 
std::string getStringMgrType () override
 
size_t getNumChunks () override
 
void deleteBuffer (const ChunkKey &key, const bool purge=true) override
 
void deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
 
Data_Namespace::AbstractBufferputBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *srcBuffer, const size_t numBytes=0) override
 
bool isBufferOnDevice (const ChunkKey &key) override
 
std::string printSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void checkpoint (const int db_id, const int tb_id) override
 
Data_Namespace::AbstractBufferalloc (const size_t numBytes=0) override
 
void free (Data_Namespace::AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 
void removeTableRelatedDS (const int db_id, const int table_id) override
 

Private Attributes

PersistentForeignStorageInterfacepersistent_foreign_storage_
 
std::map< ChunkKey,
std::unique_ptr
< ForeignStorageBuffer > > 
chunk_index_
 
mapd_shared_mutex chunk_index_mutex_
 

Detailed Description

Definition at line 120 of file ForeignStorageInterface.h.

Constructor & Destructor Documentation

ForeignStorageBufferMgr::ForeignStorageBufferMgr ( const int  db_id,
const int  table_id,
PersistentForeignStorageInterface persistent_foreign_storage 
)

Definition at line 51 of file ForeignStorageInterface.cpp.

55  : AbstractBufferMgr(0), persistent_foreign_storage_(persistent_foreign_storage) {}
PersistentForeignStorageInterface * persistent_foreign_storage_

Member Function Documentation

Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::alloc ( const size_t  numBytes = 0)
inlineoverride

Definition at line 197 of file ForeignStorageInterface.h.

References CHECK.

197  {
198  CHECK(false);
199  return nullptr;
200  }
#define CHECK(condition)
Definition: Logger.h:211
void ForeignStorageBufferMgr::checkpoint ( )
override

Definition at line 57 of file ForeignStorageInterface.cpp.

References PersistentForeignStorageInterface::append(), chunk_index_, and persistent_foreign_storage_.

57  {
58  // TODO(alex)
59  std::vector<ForeignStorageColumnBuffer> column_buffers;
60  for (auto& kv : chunk_index_) {
61  const auto buffer = kv.second->moveBuffer();
62  column_buffers.emplace_back(
63  ForeignStorageColumnBuffer{kv.first, kv.second->getSqlType(), buffer});
64  }
65  persistent_foreign_storage_->append(column_buffers);
66 }
PersistentForeignStorageInterface * persistent_foreign_storage_
virtual void append(const std::vector< ForeignStorageColumnBuffer > &column_buffers)=0
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::checkpoint ( const int  db_id,
const int  tb_id 
)
inlineoverride

Definition at line 194 of file ForeignStorageInterface.h.

References CHECK.

194 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:211
Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::createBuffer ( const ChunkKey key,
const size_t  pageSize = 0,
const size_t  initialSize = 0 
)
override

Definition at line 68 of file ForeignStorageInterface.cpp.

References chunk_index_, chunk_index_mutex_, and persistent_foreign_storage_.

71  {
72  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
73  const auto it_ok = chunk_index_.emplace(
74  key, std::make_unique<ForeignStorageBuffer>(key, persistent_foreign_storage_));
75  // this check fails if we create table, drop it and create again
76  // CHECK(it_ok.second);
77  return it_ok.first->second.get();
78 }
PersistentForeignStorageInterface * persistent_foreign_storage_
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
void ForeignStorageBufferMgr::deleteBuffer ( const ChunkKey key,
const bool  purge = true 
)
inlineoverride

Definition at line 148 of file ForeignStorageInterface.h.

References CHECK.

148  {
149  CHECK(false);
150  }
#define CHECK(condition)
Definition: Logger.h:211
void ForeignStorageBufferMgr::deleteBuffersWithPrefix ( const ChunkKey keyPrefix,
const bool  purge = true 
)
inlineoverride

Definition at line 152 of file ForeignStorageInterface.h.

References CHECK.

153  {
154  CHECK(false);
155  }
#define CHECK(condition)
Definition: Logger.h:211
void ForeignStorageBufferMgr::fetchBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer destBuffer,
const size_t  numBytes = 0 
)
override

Definition at line 89 of file ForeignStorageInterface.cpp.

References CHECK, getBuffer(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::setMemoryPtr(), Data_Namespace::AbstractBuffer::setSize(), and Data_Namespace::AbstractBuffer::syncEncoder().

91  {
92  CHECK(numBytes);
93  auto file_buffer = dynamic_cast<ForeignStorageBuffer*>(getBuffer(key, numBytes));
94  CHECK(file_buffer);
95 
96  // TODO: check if GPU is used
97  auto buf = file_buffer->tryZeroCopy(numBytes);
98  if (buf) {
99  destBuffer->setMemoryPtr(buf);
100  } else {
101  destBuffer->reserve(numBytes);
102  file_buffer->read(destBuffer->getMemoryPtr(), numBytes);
103  }
104  destBuffer->setSize(numBytes);
105  destBuffer->syncEncoder(file_buffer);
106 }
void syncEncoder(const AbstractBuffer *src_buffer)
virtual int8_t * getMemoryPtr()=0
virtual void setMemoryPtr(int8_t *new_ptr)
Data_Namespace::AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
void setSize(const size_t size)
#define CHECK(condition)
Definition: Logger.h:211
virtual void reserve(size_t num_bytes)=0

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::free ( Data_Namespace::AbstractBuffer buffer)
inlineoverride

Definition at line 202 of file ForeignStorageInterface.h.

References CHECK.

202 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:211
size_t ForeignStorageBufferMgr::getAllocated ( )
inlineoverride

Definition at line 184 of file ForeignStorageInterface.h.

References CHECK.

184  {
185  CHECK(false);
186  return 0;
187  }
#define CHECK(condition)
Definition: Logger.h:211
Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::getBuffer ( const ChunkKey key,
const size_t  numBytes = 0 
)
override

Definition at line 80 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, and chunk_index_mutex_.

Referenced by fetchBuffer().

82  {
83  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
84  const auto it = chunk_index_.find(key);
85  CHECK(it != chunk_index_.end());
86  return it->second.get();
87 }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
#define CHECK(condition)
Definition: Logger.h:211

+ Here is the caller graph for this function:

void ForeignStorageBufferMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunkMetadataVec,
const ChunkKey keyPrefix 
)
override

Definition at line 108 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, chunk_index_mutex_, and run_benchmark_import::type.

110  {
111  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(
112  chunk_index_mutex_); // is this guarding the right structure? it look slike we
113  // oly read here for chunk
114  auto chunk_it = chunk_index_.lower_bound(keyPrefix);
115  if (chunk_it == chunk_index_.end()) {
116  CHECK(false); // throw?
117  }
118 
119  while (chunk_it != chunk_index_.end() &&
120  std::search(chunk_it->first.begin(),
121  chunk_it->first.begin() + keyPrefix.size(),
122  keyPrefix.begin(),
123  keyPrefix.end()) != chunk_it->first.begin() + keyPrefix.size()) {
124  const auto& chunk_key = chunk_it->first;
125  if (chunk_key.size() == 5) {
126  if (chunk_key[4] == 1) {
127  const auto& buffer = *chunk_it->second;
128  auto type = buffer.getSqlType();
129  auto size = buffer.size();
130  auto subkey = chunk_key;
131  subkey[4] = 2;
132  auto& index_buf = *(chunk_index_.find(subkey)->second);
133  auto bs = index_buf.size() / index_buf.getSqlType().get_size();
134  auto chunk_metadata =
135  std::make_shared<ChunkMetadata>(type, size, bs, ChunkStats{});
136  chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
137  }
138  } else {
139  const auto& buffer = *chunk_it->second;
140  auto chunk_metadata = std::make_shared<ChunkMetadata>();
141  chunk_metadata->sqlType = buffer.getSqlType();
142  buffer.getEncoder()->getMetadata(chunk_metadata);
143  chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
144  }
145  chunk_it++;
146  }
147 }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
#define CHECK(condition)
Definition: Logger.h:211
size_t ForeignStorageBufferMgr::getInUseSize ( )
inlineoverride

Definition at line 179 of file ForeignStorageInterface.h.

References CHECK.

179  {
180  CHECK(false);
181  return 0;
182  }
#define CHECK(condition)
Definition: Logger.h:211
size_t ForeignStorageBufferMgr::getMaxSize ( )
inlineoverride

Definition at line 174 of file ForeignStorageInterface.h.

References CHECK.

174  {
175  CHECK(false);
176  return 0;
177  }
#define CHECK(condition)
Definition: Logger.h:211
MgrType ForeignStorageBufferMgr::getMgrType ( )
inlineoverride

Definition at line 204 of file ForeignStorageInterface.h.

References CHECK.

204  {
205  CHECK(false);
206  return FILE_MGR;
207  }
#define CHECK(condition)
Definition: Logger.h:211
size_t ForeignStorageBufferMgr::getNumChunks ( )
inlineoverride

Definition at line 143 of file ForeignStorageInterface.h.

References chunk_index_, and chunk_index_mutex_.

143  {
144  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
145  return chunk_index_.size();
146  }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
std::string ForeignStorageBufferMgr::getStringMgrType ( )
inlineoverride

Definition at line 141 of file ForeignStorageInterface.h.

141 { return ToString(FILE_MGR); }
bool ForeignStorageBufferMgr::isAllocationCapped ( )
inlineoverride

Definition at line 189 of file ForeignStorageInterface.h.

References CHECK.

189  {
190  CHECK(false);
191  return false;
192  }
#define CHECK(condition)
Definition: Logger.h:211
bool ForeignStorageBufferMgr::isBufferOnDevice ( const ChunkKey key)
inlineoverride

Definition at line 164 of file ForeignStorageInterface.h.

References CHECK.

164  {
165  CHECK(false);
166  return false;
167  }
#define CHECK(condition)
Definition: Logger.h:211
std::string ForeignStorageBufferMgr::printSlabs ( )
inlineoverride

Definition at line 169 of file ForeignStorageInterface.h.

References CHECK.

169  {
170  CHECK(false);
171  return "";
172  }
#define CHECK(condition)
Definition: Logger.h:211
Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::putBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer srcBuffer,
const size_t  numBytes = 0 
)
inlineoverride

Definition at line 157 of file ForeignStorageInterface.h.

References CHECK.

159  {
160  CHECK(false);
161  return nullptr;
162  }
#define CHECK(condition)
Definition: Logger.h:211
void ForeignStorageBufferMgr::removeTableRelatedDS ( const int  db_id,
const int  table_id 
)
inlineoverride

Definition at line 209 of file ForeignStorageInterface.h.

References PersistentForeignStorageInterface::dropTable(), and persistent_foreign_storage_.

209  {
210  persistent_foreign_storage_->dropTable(db_id, table_id);
211  }
PersistentForeignStorageInterface * persistent_foreign_storage_
virtual void dropTable(const int db_id, const int table_id)=0

+ Here is the call graph for this function:

Member Data Documentation

std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer> > ForeignStorageBufferMgr::chunk_index_
private
mapd_shared_mutex ForeignStorageBufferMgr::chunk_index_mutex_
mutableprivate
PersistentForeignStorageInterface* ForeignStorageBufferMgr::persistent_foreign_storage_
private

Definition at line 214 of file ForeignStorageInterface.h.

Referenced by checkpoint(), createBuffer(), and removeTableRelatedDS().


The documentation for this class was generated from the following files: