OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
TypedParquetStorageBuffer.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include "DataMgr/AbstractBuffer.h"
20 
21 namespace foreign_storage {
22 
23 // TODO: `TypedParquetStorageBuffer` should not extend `AbstractBuffer`, it
24 // does so here for convenience. If this class lifetime becomes somewhat
25 // permanent it should be refactored to not extend `AbstractBuffer`.
26 template <typename Type>
28  public:
30 
31  void read(int8_t* const destination,
32  const size_t num_bytes,
33  const size_t offset = 0,
34  const Data_Namespace::MemoryLevel destination_buffer_type =
36  const int destination_device_id = -1) override;
37 
38  void write(
39  int8_t* source,
40  const size_t num_bytes,
41  const size_t offset = 0,
42  const Data_Namespace::MemoryLevel source_buffer_type = Data_Namespace::CPU_LEVEL,
43  const int source_device_id = -1) override;
44 
45  void reserve(size_t additional_num_bytes) override;
46 
47  void append(
48  int8_t* source,
49  const size_t num_bytes,
50  const Data_Namespace::MemoryLevel source_buffer_type = Data_Namespace::CPU_LEVEL,
51  const int device_id = -1) override;
52 
53  void reserveNumElements(size_t additional_num_elements);
54  void appendElement(const Type& element);
55 
56  std::vector<Type>* getBufferPtr();
57 
58  int8_t* getMemoryPtr() override;
59  size_t pageCount() const override;
60  size_t pageSize() const override;
61  size_t reservedSize() const override;
62  Data_Namespace::MemoryLevel getType() const override;
63 
64  template <typename FindContainer>
65  void eraseInvalidData(const FindContainer& invalid_indices) {
66  if (invalid_indices.empty()) {
67  return;
68  }
69  auto end_it = std::remove_if(buffer_.begin(), buffer_.end(), [&](const Type& value) {
70  const Type* start = buffer_.data();
71  auto index = std::distance(start, &value);
72  return invalid_indices.find(index) != invalid_indices.end();
73  });
74  buffer_.erase(end_it, buffer_.end());
75  }
76 
77  private:
78  std::vector<Type> buffer_;
79 };
80 
81 } // namespace foreign_storage
void eraseInvalidData(const FindContainer &invalid_indices)
void append(int8_t *source, const size_t num_bytes, const Data_Namespace::MemoryLevel source_buffer_type=Data_Namespace::CPU_LEVEL, const int device_id=-1) override
void read(int8_t *const destination, const size_t num_bytes, const size_t offset=0, const Data_Namespace::MemoryLevel destination_buffer_type=Data_Namespace::CPU_LEVEL, const int destination_device_id=-1) override
void reserve(size_t additional_num_bytes) override
An AbstractBuffer is a unit of data management for a data manager.
void write(int8_t *source, const size_t num_bytes, const size_t offset=0, const Data_Namespace::MemoryLevel source_buffer_type=Data_Namespace::CPU_LEVEL, const int source_device_id=-1) override
void reserveNumElements(size_t additional_num_elements)
Data_Namespace::MemoryLevel getType() const override