OmniSciDB  0fdbebe030
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ForeignStorageBufferMgr Class Reference

#include <ForeignStorageInterface.h>

+ Inheritance diagram for ForeignStorageBufferMgr:
+ Collaboration diagram for ForeignStorageBufferMgr:

Public Member Functions

 ForeignStorageBufferMgr (const int db_id, const int table_id, PersistentForeignStorageInterface *persistent_foreign_storage)
 
void checkpoint () override
 
Data_Namespace::AbstractBuffercreateBuffer (const ChunkKey &key, const size_t pageSize=0, const size_t initialSize=0) override
 
Data_Namespace::AbstractBuffergetBuffer (const ChunkKey &key, const size_t numBytes=0) override
 
void fetchBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *destBuffer, const size_t numBytes=0) override
 
void getChunkMetadataVecForKeyPrefix (std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadataVec, const ChunkKey &keyPrefix) override
 
std::string getStringMgrType () override
 
size_t getNumChunks () override
 
void deleteBuffer (const ChunkKey &key, const bool purge=true) override
 
void deleteBuffersWithPrefix (const ChunkKey &keyPrefix, const bool purge=true) override
 
Data_Namespace::AbstractBufferputBuffer (const ChunkKey &key, Data_Namespace::AbstractBuffer *srcBuffer, const size_t numBytes=0) override
 
void getChunkMetadataVec (std::vector< std::pair< ChunkKey, ChunkMetadata >> &chunkMetadata) override
 
bool isBufferOnDevice (const ChunkKey &key) override
 
std::string printSlabs () override
 
void clearSlabs () override
 
size_t getMaxSize () override
 
size_t getInUseSize () override
 
size_t getAllocated () override
 
bool isAllocationCapped () override
 
void checkpoint (const int db_id, const int tb_id) override
 
Data_Namespace::AbstractBufferalloc (const size_t numBytes=0) override
 
void free (Data_Namespace::AbstractBuffer *buffer) override
 
MgrType getMgrType () override
 

Private Attributes

PersistentForeignStorageInterfacepersistent_foreign_storage_
 
std::map< ChunkKey,
std::unique_ptr
< ForeignStorageBuffer > > 
chunk_index_
 
mapd_shared_mutex chunk_index_mutex_
 

Detailed Description

Definition at line 112 of file ForeignStorageInterface.h.

Constructor & Destructor Documentation

ForeignStorageBufferMgr::ForeignStorageBufferMgr ( const int  db_id,
const int  table_id,
PersistentForeignStorageInterface persistent_foreign_storage 
)

Definition at line 47 of file ForeignStorageInterface.cpp.

51  : AbstractBufferMgr(0), persistent_foreign_storage_(persistent_foreign_storage) {}
PersistentForeignStorageInterface * persistent_foreign_storage_

Member Function Documentation

Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::alloc ( const size_t  numBytes = 0)
inlineoverride

Definition at line 197 of file ForeignStorageInterface.h.

References CHECK().

197  {
198  CHECK(false);
199  return nullptr;
200  }
CHECK(cgen_state)

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::checkpoint ( )
override

Definition at line 53 of file ForeignStorageInterface.cpp.

References PersistentForeignStorageInterface::append(), chunk_index_, persistent_foreign_storage_, and ForeignStorageColumnBuffer::sql_type.

53  {
54  // TODO(alex)
55  std::vector<ForeignStorageColumnBuffer> column_buffers;
56  for (auto& kv : chunk_index_) {
57  const auto buffer = kv.second->moveBuffer();
58  column_buffers.emplace_back(
59  ForeignStorageColumnBuffer{kv.first, kv.second->sql_type, buffer});
60  }
61  persistent_foreign_storage_->append(column_buffers);
62 }
PersistentForeignStorageInterface * persistent_foreign_storage_
virtual void append(const std::vector< ForeignStorageColumnBuffer > &column_buffers)=0
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::checkpoint ( const int  db_id,
const int  tb_id 
)
inlineoverride

Definition at line 194 of file ForeignStorageInterface.h.

References CHECK().

194 { CHECK(false); }
CHECK(cgen_state)

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::clearSlabs ( )
inlineoverride

Definition at line 172 of file ForeignStorageInterface.h.

References CHECK().

172 { CHECK(false); }
CHECK(cgen_state)

+ Here is the call graph for this function:

Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::createBuffer ( const ChunkKey key,
const size_t  pageSize = 0,
const size_t  initialSize = 0 
)
override

Definition at line 64 of file ForeignStorageInterface.cpp.

References chunk_index_, chunk_index_mutex_, and persistent_foreign_storage_.

67  {
68  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
69  const auto it_ok = chunk_index_.emplace(
70  key, std::make_unique<ForeignStorageBuffer>(key, persistent_foreign_storage_));
71  // this check fails if we create table, drop it and create again
72  // CHECK(it_ok.second);
73  return it_ok.first->second.get();
74 }
PersistentForeignStorageInterface * persistent_foreign_storage_
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
void ForeignStorageBufferMgr::deleteBuffer ( const ChunkKey key,
const bool  purge = true 
)
inlineoverride

Definition at line 141 of file ForeignStorageInterface.h.

References CHECK().

141  {
142  CHECK(false);
143  }
CHECK(cgen_state)

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::deleteBuffersWithPrefix ( const ChunkKey keyPrefix,
const bool  purge = true 
)
inlineoverride

Definition at line 145 of file ForeignStorageInterface.h.

References CHECK().

146  {
147  CHECK(false);
148  }
CHECK(cgen_state)

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::fetchBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer destBuffer,
const size_t  numBytes = 0 
)
override

Definition at line 85 of file ForeignStorageInterface.cpp.

References CHECK(), getBuffer(), Data_Namespace::AbstractBuffer::getMemoryPtr(), Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::setSize(), and Data_Namespace::AbstractBuffer::syncEncoder().

87  {
88  CHECK(numBytes);
89  destBuffer->reserve(numBytes);
90  auto file_buffer = getBuffer(key, numBytes);
91  file_buffer->read(destBuffer->getMemoryPtr(), numBytes);
92  destBuffer->setSize(numBytes);
93  destBuffer->syncEncoder(file_buffer);
94 }
void syncEncoder(const AbstractBuffer *src_buffer)
virtual int8_t * getMemoryPtr()=0
CHECK(cgen_state)
Data_Namespace::AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
void setSize(const size_t size)
virtual void reserve(size_t num_bytes)=0

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::free ( Data_Namespace::AbstractBuffer buffer)
inlineoverride

Definition at line 202 of file ForeignStorageInterface.h.

References CHECK().

202 { CHECK(false); }
CHECK(cgen_state)

+ Here is the call graph for this function:

size_t ForeignStorageBufferMgr::getAllocated ( )
inlineoverride

Definition at line 184 of file ForeignStorageInterface.h.

References CHECK().

184  {
185  CHECK(false);
186  return 0;
187  }
CHECK(cgen_state)

+ Here is the call graph for this function:

Data_Namespace::AbstractBuffer * ForeignStorageBufferMgr::getBuffer ( const ChunkKey key,
const size_t  numBytes = 0 
)
override

Definition at line 76 of file ForeignStorageInterface.cpp.

References CHECK(), chunk_index_, and chunk_index_mutex_.

Referenced by fetchBuffer().

78  {
79  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
80  const auto it = chunk_index_.find(key);
81  CHECK(it != chunk_index_.end());
82  return it->second.get();
83 }
CHECK(cgen_state)
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void ForeignStorageBufferMgr::getChunkMetadataVec ( std::vector< std::pair< ChunkKey, ChunkMetadata >> &  chunkMetadata)
inlineoverride

Definition at line 157 of file ForeignStorageInterface.h.

References CHECK().

158  {
159  CHECK(false);
160  }
CHECK(cgen_state)

+ Here is the call graph for this function:

void ForeignStorageBufferMgr::getChunkMetadataVecForKeyPrefix ( std::vector< std::pair< ChunkKey, ChunkMetadata >> &  chunkMetadataVec,
const ChunkKey keyPrefix 
)
override

Definition at line 96 of file ForeignStorageInterface.cpp.

References CHECK(), chunk_index_, chunk_index_mutex_, and run_benchmark_import::type.

98  {
99  mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(
100  chunk_index_mutex_); // is this guarding the right structure? it look slike we
101  // oly read here for chunk
102  auto chunk_it = chunk_index_.lower_bound(keyPrefix);
103  if (chunk_it == chunk_index_.end()) {
104  CHECK(false); // throw?
105  }
106 
107  while (chunk_it != chunk_index_.end() &&
108  std::search(chunk_it->first.begin(),
109  chunk_it->first.begin() + keyPrefix.size(),
110  keyPrefix.begin(),
111  keyPrefix.end()) != chunk_it->first.begin() + keyPrefix.size()) {
112  const auto& chunk_key = chunk_it->first;
113  if (chunk_key.size() == 5) {
114  if (chunk_key[4] == 1) {
115  const auto& buffer = *chunk_it->second;
116  auto type = buffer.sql_type;
117  auto size = buffer.size();
118  auto subkey = chunk_key;
119  subkey[4] = 2;
120  auto& index_buf = *(chunk_index_.find(subkey)->second);
121  auto bs = index_buf.size() / index_buf.sql_type.get_size();
122  ChunkMetadata m{type, size, bs, ChunkStats{}};
123  chunkMetadataVec.emplace_back(chunk_key, m);
124  }
125  } else {
126  const auto& buffer = *chunk_it->second;
127  ChunkMetadata m{buffer.sql_type};
128  buffer.encoder->getMetadata(m);
129  chunkMetadataVec.emplace_back(chunk_key, m);
130  }
131  chunk_it++;
132  }
133 }
CHECK(cgen_state)
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_

+ Here is the call graph for this function:

size_t ForeignStorageBufferMgr::getInUseSize ( )
inlineoverride

Definition at line 179 of file ForeignStorageInterface.h.

References CHECK().

179  {
180  CHECK(false);
181  return 0;
182  }
CHECK(cgen_state)

+ Here is the call graph for this function:

size_t ForeignStorageBufferMgr::getMaxSize ( )
inlineoverride

Definition at line 174 of file ForeignStorageInterface.h.

References CHECK().

174  {
175  CHECK(false);
176  return 0;
177  }
CHECK(cgen_state)

+ Here is the call graph for this function:

MgrType ForeignStorageBufferMgr::getMgrType ( )
inlineoverride

Definition at line 204 of file ForeignStorageInterface.h.

References CHECK().

204  {
205  CHECK(false);
206  return FILE_MGR;
207  }
CHECK(cgen_state)

+ Here is the call graph for this function:

size_t ForeignStorageBufferMgr::getNumChunks ( )
inlineoverride

Definition at line 136 of file ForeignStorageInterface.h.

References chunk_index_, and chunk_index_mutex_.

136  {
137  mapd_shared_lock<mapd_shared_mutex> chunk_index_write_lock(chunk_index_mutex_);
138  return chunk_index_.size();
139  }
mapd_shared_mutex chunk_index_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
std::string ForeignStorageBufferMgr::getStringMgrType ( )
inlineoverride

Definition at line 134 of file ForeignStorageInterface.h.

134 { return ToString(FILE_MGR); }
bool ForeignStorageBufferMgr::isAllocationCapped ( )
inlineoverride

Definition at line 189 of file ForeignStorageInterface.h.

References CHECK().

189  {
190  CHECK(false);
191  return false;
192  }
CHECK(cgen_state)

+ Here is the call graph for this function:

bool ForeignStorageBufferMgr::isBufferOnDevice ( const ChunkKey key)
inlineoverride

Definition at line 162 of file ForeignStorageInterface.h.

References CHECK().

162  {
163  CHECK(false);
164  return false;
165  }
CHECK(cgen_state)

+ Here is the call graph for this function:

std::string ForeignStorageBufferMgr::printSlabs ( )
inlineoverride

Definition at line 167 of file ForeignStorageInterface.h.

References CHECK().

167  {
168  CHECK(false);
169  return "";
170  }
CHECK(cgen_state)

+ Here is the call graph for this function:

Data_Namespace::AbstractBuffer* ForeignStorageBufferMgr::putBuffer ( const ChunkKey key,
Data_Namespace::AbstractBuffer srcBuffer,
const size_t  numBytes = 0 
)
inlineoverride

Definition at line 150 of file ForeignStorageInterface.h.

References CHECK().

152  {
153  CHECK(false);
154  return nullptr;
155  }
CHECK(cgen_state)

+ Here is the call graph for this function:

Member Data Documentation

std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer> > ForeignStorageBufferMgr::chunk_index_
private
mapd_shared_mutex ForeignStorageBufferMgr::chunk_index_mutex_
mutableprivate
PersistentForeignStorageInterface* ForeignStorageBufferMgr::persistent_foreign_storage_
private

Definition at line 210 of file ForeignStorageInterface.h.

Referenced by checkpoint(), and createBuffer().


The documentation for this class was generated from the following files: