OmniSciDB  95562058bd
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ForeignStorageBufferMgr Class Reference

#include <ForeignStorageInterface.h>

+ Inheritance diagram for ForeignStorageBufferMgr:
+ Collaboration diagram for ForeignStorageBufferMgr:

Public Member Functions

 ForeignStorageBufferMgr (const int db_id, const int table_id, PersistentForeignStorageInterface *persistent_foreign_storage)
 
void checkpoint () override
 
Data_Namespace::AbstractBuffercreateBuffer (const ChunkKey &key, const size_t pageSize=0, const size_t initialSize=0) override
 
Data_Namespace::AbstractBuffergetBuffer (const ChunkKey &key, const size_t numBytes=0) override
 
void fetchBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *destBuffer, const size_t numBytes=0) override
 
void getChunkMetadataVecForKeyPrefix (ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
 
std::string getStringMgrType () override
 
size_t getNumChunks () override
 
void deleteBuffer (const ChunkKey &key, const bool purge=true) override
 
void deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
 
Data_Namespace::AbstractBufferputBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *srcBuffer, const size_t numBytes=0) override
 
bool isBufferOnDevice (const ChunkKey &key) override
 
std::string printSlabs () override
 
void clearSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void checkpoint (const int db_id, const int tb_id) override
 
Data_Namespace::AbstractBufferalloc (const size_t numBytes=0) override
 
void free (Data_Namespace::AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 
void removeTableRelatedDS (const int db_id, const int table_id) override
 

Private Attributes

PersistentForeignStorageInterfacepersistent_foreign_storage_
 
std::map< ChunkKey,
std::unique_ptr
< ForeignStorageBuffer > > 
chunk_index_
 
mapd_shared_mutex chunk_index_mutex_
 

Detailed Description

Definition at line 110 of file ForeignStorageInterface.h.

Constructor & Destructor Documentation

ForeignStorageBufferMgr::ForeignStorageBufferMgr ( const int  db_id,
const int  table_id,
PersistentForeignStorageInterface persistent_foreign_storage 
)

Definition at line 46 of file ForeignStorageInterface.cpp.

50  : AbstractBufferMgr(0), persistent_foreign_storage_(persistent_foreign_storage) {}
PersistentForeignStorageInterface * persistent_foreign_storage_

Member Function Documentation

Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::alloc ( const size_t  numBytes = 0)
inlineoverride

Definition at line 189 of file ForeignStorageInterface.h.

References CHECK.

189  {
190  CHECK(false);
191  return nullptr;
192  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::checkpoint ( )
override

Definition at line 52 of file ForeignStorageInterface.cpp.

References PersistentForeignStorageInterface::append(), chunk_index_, and persistent_foreign_storage_.

52  {
53  // TODO(alex)
54  std::vector<ForeignStorageColumnBuffer> column_buffers;
55  for (auto& kv : chunk_index_) {
56  const auto buffer = kv.second->moveBuffer();
57  column_buffers.emplace_back(
58  ForeignStorageColumnBuffer{kv.first, kv.second->getSqlType(), buffer});
59  }
60  persistent_foreign_storage_->append(column_buffers);
61 }
PersistentForeignStorageInterface * persistent_foreign_storage_
virtual void append(const std::vector< ForeignStorageColumnBuffer > &column_buffers)=0
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::checkpoint ( const int  db_id,
const int  tb_id 
)
inlineoverride

Definition at line 186 of file ForeignStorageInterface.h.

References CHECK.

186 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::clearSlabs ( )
inlineoverride

Definition at line 164 of file ForeignStorageInterface.h.

References CHECK.

164 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:197
Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::createBuffer ( const ChunkKey key,
const size_t  pageSize = 0,
const size_t  initialSize = 0 
)
override

Definition at line 63 of file ForeignStorageInterface.cpp.

References chunk_index_, chunk_index_mutex_, and persistent_foreign_storage_.

66  {
67  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
68  const auto it_ok = chunk_index_.emplace(
69  key, std::make_unique<ForeignStorageBuffer>(key, persistent_foreign_storage_));
70  // this check fails if we create table, drop it and create again
71  // CHECK(it_ok.second);
72  return it_ok.first->second.get();
73 }
PersistentForeignStorageInterface * persistent_foreign_storage_
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
void ForeignStorageBufferMgr::deleteBuffer ( const ChunkKey key,
const bool  purge = true 
)
inlineoverride

Definition at line 138 of file ForeignStorageInterface.h.

References CHECK.

138  {
139  CHECK(false);
140  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::deleteBuffersWithPrefix ( const ChunkKey keyPrefix,
const bool  purge = true 
)
inlineoverride

Definition at line 142 of file ForeignStorageInterface.h.

References CHECK.

143  {
144  CHECK(false);
145  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::fetchBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer destBuffer,
const size_t  numBytes = 0 
)
override

Definition at line 84 of file ForeignStorageInterface.cpp.

References CHECK, getBuffer(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::setSize(), and Data_Namespace::AbstractBuffer::syncEncoder().

86  {
87  CHECK(numBytes);
88  destBuffer->reserve(numBytes);
89  auto file_buffer = getBuffer(key, numBytes);
90  file_buffer->read(destBuffer->getMemoryPtr(), numBytes);
91  destBuffer->setSize(numBytes);
92  destBuffer->syncEncoder(file_buffer);
93 }
void syncEncoder(const AbstractBuffer *src_buffer)
virtual int8_t * getMemoryPtr()=0
Data_Namespace::AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
void setSize(const size_t size)
#define CHECK(condition)
Definition: Logger.h:197
virtual void reserve(size_t num_bytes)=0

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::free ( Data_Namespace::AbstractBuffer buffer)
inlineoverride

Definition at line 194 of file ForeignStorageInterface.h.

References CHECK.

194 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getAllocated ( )
inlineoverride

Definition at line 176 of file ForeignStorageInterface.h.

References CHECK.

176  {
177  CHECK(false);
178  return 0;
179  }
#define CHECK(condition)
Definition: Logger.h:197
Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::getBuffer ( const ChunkKey key,
const size_t  numBytes = 0 
)
override

Definition at line 75 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, and chunk_index_mutex_.

Referenced by fetchBuffer().

77  {
78  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
79  const auto it = chunk_index_.find(key);
80  CHECK(it != chunk_index_.end());
81  return it->second.get();
82 }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the caller graph for this function:

void ForeignStorageBufferMgr::getChunkMetadataVecForKeyPrefix ( ChunkMetadataVector chunkMetadataVec,
const ChunkKey keyPrefix 
)
override

Definition at line 95 of file ForeignStorageInterface.cpp.

References CHECK, chunk_index_, chunk_index_mutex_, and run_benchmark_import::type.

97  {
98  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(
99  chunk_index_mutex_); // is this guarding the right structure? it look slike we
100  // oly read here for chunk
101  auto chunk_it = chunk_index_.lower_bound(keyPrefix);
102  if (chunk_it == chunk_index_.end()) {
103  CHECK(false); // throw?
104  }
105 
106  while (chunk_it != chunk_index_.end() &&
107  std::search(chunk_it->first.begin(),
108  chunk_it->first.begin() + keyPrefix.size(),
109  keyPrefix.begin(),
110  keyPrefix.end()) != chunk_it->first.begin() + keyPrefix.size()) {
111  const auto& chunk_key = chunk_it->first;
112  if (chunk_key.size() == 5) {
113  if (chunk_key[4] == 1) {
114  const auto& buffer = *chunk_it->second;
115  auto type = buffer.getSqlType();
116  auto size = buffer.size();
117  auto subkey = chunk_key;
118  subkey[4] = 2;
119  auto& index_buf = *(chunk_index_.find(subkey)->second);
120  auto bs = index_buf.size() / index_buf.getSqlType().get_size();
121  auto chunk_metadata =
122  std::make_shared<ChunkMetadata>(type, size, bs, ChunkStats{});
123  chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
124  }
125  } else {
126  const auto& buffer = *chunk_it->second;
127  auto chunk_metadata = std::make_shared<ChunkMetadata>();
128  chunk_metadata->sqlType = buffer.getSqlType();
129  buffer.getEncoder()->getMetadata(chunk_metadata);
130  chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
131  }
132  chunk_it++;
133  }
134 }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getInUseSize ( )
inlineoverride

Definition at line 171 of file ForeignStorageInterface.h.

References CHECK.

171  {
172  CHECK(false);
173  return 0;
174  }
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getMaxSize ( )
inlineoverride

Definition at line 166 of file ForeignStorageInterface.h.

References CHECK.

166  {
167  CHECK(false);
168  return 0;
169  }
#define CHECK(condition)
Definition: Logger.h:197
MgrType ForeignStorageBufferMgr::getMgrType ( )
inlineoverride

Definition at line 196 of file ForeignStorageInterface.h.

References CHECK.

196  {
197  CHECK(false);
198  return FILE_MGR;
199  }
#define CHECK(condition)
Definition: Logger.h:197
size_t ForeignStorageBufferMgr::getNumChunks ( )
inlineoverride

Definition at line 133 of file ForeignStorageInterface.h.

References chunk_index_, and chunk_index_mutex_.

133  {
134  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
135  return chunk_index_.size();
136  }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
std::string ForeignStorageBufferMgr::getStringMgrType ( )
inlineoverride

Definition at line 131 of file ForeignStorageInterface.h.

131 { return ToString(FILE_MGR); }
bool ForeignStorageBufferMgr::isAllocationCapped ( )
inlineoverride

Definition at line 181 of file ForeignStorageInterface.h.

References CHECK.

181  {
182  CHECK(false);
183  return false;
184  }
#define CHECK(condition)
Definition: Logger.h:197
bool ForeignStorageBufferMgr::isBufferOnDevice ( const ChunkKey key)
inlineoverride

Definition at line 154 of file ForeignStorageInterface.h.

References CHECK.

154  {
155  CHECK(false);
156  return false;
157  }
#define CHECK(condition)
Definition: Logger.h:197
std::string ForeignStorageBufferMgr::printSlabs ( )
inlineoverride

Definition at line 159 of file ForeignStorageInterface.h.

References CHECK.

159  {
160  CHECK(false);
161  return "";
162  }
#define CHECK(condition)
Definition: Logger.h:197
Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::putBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer srcBuffer,
const size_t  numBytes = 0 
)
inlineoverride

Definition at line 147 of file ForeignStorageInterface.h.

References CHECK.

149  {
150  CHECK(false);
151  return nullptr;
152  }
#define CHECK(condition)
Definition: Logger.h:197
void ForeignStorageBufferMgr::removeTableRelatedDS ( const int  db_id,
const int  table_id 
)
inlineoverride

Definition at line 201 of file ForeignStorageInterface.h.

References UNREACHABLE.

201  {
202  UNREACHABLE();
203  }
#define UNREACHABLE()
Definition: Logger.h:241

Member Data Documentation

std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer> > ForeignStorageBufferMgr::chunk_index_
private
mapd_shared_mutex ForeignStorageBufferMgr::chunk_index_mutex_
mutableprivate
PersistentForeignStorageInterface* ForeignStorageBufferMgr::persistent_foreign_storage_
private

Definition at line 206 of file ForeignStorageInterface.h.

Referenced by checkpoint(), and createBuffer().


The documentation for this class was generated from the following files: