24 , chunk_key_(chunk_key)
25 , persistent_foreign_storage_(persistent_foreign_storage) {}
28 const size_t numBytes,
31 const int dstDeviceId) {
42 const size_t numBytes,
46 buff_.insert(
buff_.end(), src, src + numBytes);
54 : AbstractBufferMgr(0), persistent_foreign_storage_(persistent_foreign_storage) {}
58 std::vector<ForeignStorageColumnBuffer> column_buffers;
60 const auto buffer = kv.second->moveBuffer();
61 column_buffers.emplace_back(
69 const size_t pageSize,
70 const size_t initialSize) {
76 return it_ok.first->second.get();
81 const size_t numBytes) {
85 return it->second.get();
90 const size_t numBytes) {
96 auto buf = file_buffer->tryZeroCopy(numBytes);
101 file_buffer->read(destBuffer->
getMemoryPtr(), numBytes);
110 mapd_unique_lock<mapd_shared_mutex> chunk_index_write_lock(
119 std::search(chunk_it->first.begin(),
120 chunk_it->first.begin() + keyPrefix.size(),
122 keyPrefix.end()) != chunk_it->first.begin() + keyPrefix.size()) {
123 const auto& chunk_key = chunk_it->first;
124 if (chunk_key.size() == 5) {
125 if (chunk_key[4] == 1) {
126 const auto& buffer = *chunk_it->second;
127 auto type = buffer.getSqlType();
128 auto size = buffer.size();
129 auto subkey = chunk_key;
132 auto bs = index_buf.size() / index_buf.getSqlType().get_size();
133 auto chunk_metadata =
135 chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
138 const auto& buffer = *chunk_it->second;
139 auto chunk_metadata = std::make_shared<ChunkMetadata>();
140 chunk_metadata->sqlType = buffer.getSqlType();
141 buffer.getEncoder()->getMetadata(chunk_metadata);
142 chunkMetadataVec.emplace_back(chunk_key, chunk_metadata);
150 const int table_id) {
151 auto key = std::make_pair(db_id, table_id);
156 std::lock_guard<std::mutex> persistent_storage_interfaces_lock(
163 key, std::make_unique<ForeignStorageBufferMgr>(db_id, table_id, it->second));
165 return it_ok.first->second.get();
169 std::unique_ptr<PersistentForeignStorageInterface> persistent_foreign_storage) {
170 std::lock_guard<std::mutex> persistent_storage_interfaces_lock(
173 persistent_foreign_storage->getType(), std::move(persistent_foreign_storage));
178 size_t sep = type.find_first_of(
':'), sep2 = sep != std::string::npos ? sep + 1 : sep;
179 auto res = std::make_pair(
to_upper(type.substr(0, sep)), type.substr(sep2));
185 std::list<ColumnDescriptor>& cols) {
187 std::unique_lock<std::mutex> persistent_storage_interfaces_lock(
191 throw std::runtime_error(
"storage type " +
type.first +
" not supported");
193 auto& p = it->second;
194 persistent_storage_interfaces_lock.unlock();
195 p->prepareTable(db_id,
type.second, td, cols);
200 const std::list<ColumnDescriptor>& cols) {
201 const int table_id = td.
tableId;
204 std::unique_lock<std::mutex> persistent_storage_interfaces_lock(
208 throw std::runtime_error(
"storage type " +
type.first +
" not supported");
213 std::make_pair(db_id, table_id), it->second.get());
216 persistent_storage_interfaces_lock.unlock();
217 it_ok.first->second->registerTable(catalog,
230 std::unordered_map<std::string, std::unique_ptr<PersistentForeignStorageInterface>>
234 std::map<std::pair<int, int>, std::unique_ptr<ForeignStorageBufferMgr>>
PersistentForeignStorageInterface * persistent_foreign_storage_
void getChunkMetadataVecForKeyPrefix(ChunkMetadataVector &chunkMetadataVec, const ChunkKey &keyPrefix) override
std::vector< int > ChunkKey
class for a per-database catalog. also includes metadata for the current database and the current use...
void syncEncoder(const AbstractBuffer *src_buffer)
void checkpoint() override
static Data_Namespace::AbstractBufferMgr * lookupBufferManager(const int db_id, const int table_id)
void append(int8_t *src, const size_t numBytes, const Data_Namespace::MemoryLevel srcBufferType=Data_Namespace::CPU_LEVEL, const int deviceId=-1) override
virtual int8_t * getMemoryPtr()=0
const ChunkKey chunk_key_
void fetchBuffer(const ChunkKey &key, Data_Namespace::AbstractBuffer *destBuffer, const size_t numBytes=0) override
static void registerPersistentStorageInterface(std::unique_ptr< PersistentForeignStorageInterface > persistent_foreign_storage)
const DBMetadata & getCurrentDB() const
ForeignStorageBufferMgr(const int db_id, const int table_id, PersistentForeignStorageInterface *persistent_foreign_storage)
mapd_shared_mutex chunk_index_mutex_
virtual void append(const std::vector< ForeignStorageColumnBuffer > &column_buffers)=0
ForeignStorageBuffer(const ChunkKey &chunk_key, PersistentForeignStorageInterface *persistent_foreign_storage)
An AbstractBuffer is a unit of data management for a data manager.
static std::map< std::pair< int, int >, std::unique_ptr< ForeignStorageBufferMgr > > managers_map_
static void registerTable(Catalog_Namespace::Catalog *catalog, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols)
ids are created
static std::mutex persistent_storage_interfaces_mutex_
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_index_
std::vector< int8_t > buff_
virtual void setMemoryPtr(int8_t *new_ptr)
Data_Namespace::AbstractBuffer * getBuffer(const ChunkKey &key, const size_t numBytes=0) override
virtual int8_t * tryZeroCopy(const ChunkKey &chunk_key, const SQLTypeInfo &sql_type, const size_t num_bytes)
void setSize(const size_t size)
static std::map< std::pair< int, int >, PersistentForeignStorageInterface * > table_persistent_storage_interface_map_
static std::unordered_map< std::string, std::unique_ptr< PersistentForeignStorageInterface > > persistent_storage_interfaces_
PersistentForeignStorageInterface * persistent_foreign_storage_
void read(int8_t *const dst, const size_t numBytes, const size_t offset=0, const Data_Namespace::MemoryLevel dstBufferType=Data_Namespace::CPU_LEVEL, const int dstDeviceId=-1) override
virtual void reserve(size_t num_bytes)=0
std::pair< std::string, std::string > parseStorageType(const std::string &type)
virtual void read(const ChunkKey &chunk_key, const SQLTypeInfo &sql_type, int8_t *dest, const size_t num_bytes)=0
int8_t * tryZeroCopy(const size_t numBytes)
static void prepareTable(const int db_id, TableDescriptor &td, std::list< ColumnDescriptor > &cols)
prepare table options and modify columns
Data_Namespace::AbstractBuffer * createBuffer(const ChunkKey &key, const size_t pageSize=0, const size_t initialSize=0) override