OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp} Namespace Reference

Functions

void initialize_chunks (std::map< ChunkKey, Chunk_NS::Chunk > &chunks, const ChunkToBufferMap &buffers, size_t row_count, std::set< const ColumnDescriptor * > &columns_to_parse, int32_t fragment_id, const Catalog_Namespace::Catalog &catalog)
 
void initialize_import_buffers (std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers_map, const std::set< const ColumnDescriptor * > &columns_to_parse, const Catalog_Namespace::Catalog &catalog)
 

Function Documentation

void foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_chunks ( std::map< ChunkKey, Chunk_NS::Chunk > &  chunks,
const ChunkToBufferMap &  buffers,
size_t  row_count,
std::set< const ColumnDescriptor * > &  columns_to_parse,
int32_t  fragment_id,
const Catalog_Namespace::Catalog catalog 
)

Definition at line 101 of file InternalSystemDataWrapper.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, CHUNK_KEY_VARLEN_IDX, Catalog_Namespace::Catalog::getMetadataForColumn(), is_varlen_data_key(), is_varlen_index_key(), Data_Namespace::AbstractBuffer::reserve(), and UNREACHABLE.

Referenced by foreign_storage::InternalSystemDataWrapper::populateChunkBuffers().

106  {
107  for (auto& [chunk_key, buffer] : buffers) {
108  CHECK_EQ(fragment_id, chunk_key[CHUNK_KEY_FRAGMENT_IDX]);
109  const auto column = catalog.getMetadataForColumn(chunk_key[CHUNK_KEY_TABLE_IDX],
110  chunk_key[CHUNK_KEY_COLUMN_IDX]);
111  if (is_varlen_index_key(chunk_key)) {
112  continue;
113  }
114  chunks[chunk_key] = Chunk_NS::Chunk{column};
115  if (column->columnType.is_varlen_indeed()) {
116  CHECK(is_varlen_data_key(chunk_key));
117  size_t index_offset_size{0};
118  if (column->columnType.is_string()) {
119  index_offset_size = sizeof(StringOffsetT);
120  } else if (column->columnType.is_array()) {
121  index_offset_size = sizeof(ArrayOffsetT);
122  } else {
123  UNREACHABLE() << "Unexpected column type: " << column->columnType.to_string();
124  }
125  ChunkKey index_chunk_key = chunk_key;
126  index_chunk_key[CHUNK_KEY_VARLEN_IDX] = 2;
127  CHECK(buffers.find(index_chunk_key) != buffers.end());
128  AbstractBuffer* index_buffer = buffers.find(index_chunk_key)->second;
129  index_buffer->reserve(index_offset_size * row_count + 1);
130  chunks[chunk_key].setIndexBuffer(index_buffer);
131  }
132 
133  if (!column->columnType.is_varlen_indeed()) {
134  buffer->reserve(column->columnType.get_size() * row_count);
135  }
136  chunks[chunk_key].setBuffer(buffer);
137  chunks[chunk_key].initEncoder();
138  columns_to_parse.emplace(column);
139  }
140 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
std::vector< int > ChunkKey
Definition: types.h:36
bool is_varlen_data_key(const ChunkKey &key)
Definition: types.h:75
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:41
#define UNREACHABLE()
Definition: Logger.h:266
int32_t StringOffsetT
Definition: sqltypes.h:1113
bool is_varlen_index_key(const ChunkKey &key)
Definition: types.h:79
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:39
An AbstractBuffer is a unit of data management for a data manager.
int32_t ArrayOffsetT
Definition: sqltypes.h:1114
#define CHUNK_KEY_VARLEN_IDX
Definition: types.h:42
#define CHECK(condition)
Definition: Logger.h:222
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:40
virtual void reserve(size_t num_bytes)=0

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_import_buffers ( std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
std::map< std::string, import_export::TypedImportBuffer * > &  import_buffers_map,
const std::set< const ColumnDescriptor * > &  columns_to_parse,
const Catalog_Namespace::Catalog catalog 
)

Definition at line 142 of file InternalSystemDataWrapper.cpp.

References Catalog_Namespace::Catalog::getMetadataForDict(), IS_STRING, and kENCODING_DICT.

Referenced by foreign_storage::InternalSystemDataWrapper::populateChunkBuffers().

146  {
147  for (const auto column : columns_to_parse) {
148  StringDictionary* string_dictionary = nullptr;
149  if (column->columnType.is_dict_encoded_string() ||
150  (column->columnType.is_array() && IS_STRING(column->columnType.get_subtype()) &&
151  column->columnType.get_compression() == kENCODING_DICT)) {
152  auto dict_descriptor =
153  catalog.getMetadataForDict(column->columnType.get_comp_param(), true);
154  string_dictionary = dict_descriptor->stringDict.get();
155  }
156  import_buffers.emplace_back(
157  std::make_unique<import_export::TypedImportBuffer>(column, string_dictionary));
158  import_buffers_map[column->columnName] = import_buffers.back().get();
159  }
160 }
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1960
#define IS_STRING(T)
Definition: sqltypes.h:250

+ Here is the call graph for this function:

+ Here is the caller graph for this function: