OmniSciDB  d2f719934e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp} Namespace Reference

Functions

void initialize_chunks (std::map< ChunkKey, Chunk_NS::Chunk > &chunks, const ChunkToBufferMap &buffers, size_t row_count, std::set< const ColumnDescriptor * > &columns_to_parse, int32_t fragment_id, const Catalog_Namespace::Catalog &catalog)
 
void initialize_import_buffers (std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers_map, const std::set< const ColumnDescriptor * > &columns_to_parse, const Catalog_Namespace::Catalog &catalog)
 

Function Documentation

void foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_chunks ( std::map< ChunkKey, Chunk_NS::Chunk > &  chunks,
const ChunkToBufferMap &  buffers,
size_t  row_count,
std::set< const ColumnDescriptor * > &  columns_to_parse,
int32_t  fragment_id,
const Catalog_Namespace::Catalog catalog 
)

Definition at line 62 of file InternalSystemDataWrapper.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, CHUNK_KEY_VARLEN_IDX, Catalog_Namespace::Catalog::getMetadataForColumn(), is_varlen_data_key(), is_varlen_index_key(), Data_Namespace::AbstractBuffer::reserve(), and UNREACHABLE.

Referenced by foreign_storage::InternalSystemDataWrapper::populateChunkBuffers().

67  {
68  for (auto& [chunk_key, buffer] : buffers) {
69  CHECK_EQ(fragment_id, chunk_key[CHUNK_KEY_FRAGMENT_IDX]);
70  const auto column = catalog.getMetadataForColumn(chunk_key[CHUNK_KEY_TABLE_IDX],
71  chunk_key[CHUNK_KEY_COLUMN_IDX]);
72  if (is_varlen_index_key(chunk_key)) {
73  continue;
74  }
75  chunks[chunk_key] = Chunk_NS::Chunk{column};
76  if (column->columnType.is_varlen_indeed()) {
77  CHECK(is_varlen_data_key(chunk_key));
78  size_t index_offset_size{0};
79  if (column->columnType.is_string()) {
80  index_offset_size = sizeof(StringOffsetT);
81  } else if (column->columnType.is_array()) {
82  index_offset_size = sizeof(ArrayOffsetT);
83  } else {
84  UNREACHABLE() << "Unexpected column type: " << column->columnType.to_string();
85  }
86  ChunkKey index_chunk_key = chunk_key;
87  index_chunk_key[CHUNK_KEY_VARLEN_IDX] = 2;
88  CHECK(buffers.find(index_chunk_key) != buffers.end());
89  AbstractBuffer* index_buffer = buffers.find(index_chunk_key)->second;
90  index_buffer->reserve(index_offset_size * row_count + 1);
91  chunks[chunk_key].setIndexBuffer(index_buffer);
92  }
93 
94  if (!column->columnType.is_varlen_indeed()) {
95  buffer->reserve(column->columnType.get_size() * row_count);
96  }
97  chunks[chunk_key].setBuffer(buffer);
98  chunks[chunk_key].initEncoder();
99  columns_to_parse.emplace(column);
100  }
101 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
std::vector< int > ChunkKey
Definition: types.h:37
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:71
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
#define UNREACHABLE()
Definition: Logger.h:255
int32_t StringOffsetT
Definition: sqltypes.h:1090
bool is_varlen_index_key(const ChunkKey &key)
Definition: types.h:75
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:40
An AbstractBuffer is a unit of data management for a data manager.
int32_t ArrayOffsetT
Definition: sqltypes.h:1091
#define CHUNK_KEY_VARLEN_IDX
Definition: types.h:43
#define CHECK(condition)
Definition: Logger.h:211
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
virtual void reserve(size_t num_bytes)=0

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_import_buffers ( std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
std::map< std::string, import_export::TypedImportBuffer * > &  import_buffers_map,
const std::set< const ColumnDescriptor * > &  columns_to_parse,
const Catalog_Namespace::Catalog catalog 
)

Definition at line 103 of file InternalSystemDataWrapper.cpp.

References Catalog_Namespace::Catalog::getMetadataForDict(), IS_STRING, and kENCODING_DICT.

Referenced by foreign_storage::InternalSystemDataWrapper::populateChunkBuffers().

107  {
108  for (const auto column : columns_to_parse) {
109  StringDictionary* string_dictionary = nullptr;
110  if (column->columnType.is_dict_encoded_string() ||
111  (column->columnType.is_array() && IS_STRING(column->columnType.get_subtype()) &&
112  column->columnType.get_compression() == kENCODING_DICT)) {
113  auto dict_descriptor =
114  catalog.getMetadataForDict(column->columnType.get_comp_param(), true);
115  string_dictionary = dict_descriptor->stringDict.get();
116  }
117  import_buffers.emplace_back(
118  std::make_unique<import_export::TypedImportBuffer>(column, string_dictionary));
119  import_buffers_map[column->columnName] = import_buffers.back().get();
120  }
121 }
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1554
#define IS_STRING(T)
Definition: sqltypes.h:250

+ Here is the call graph for this function:

+ Here is the caller graph for this function: