OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
DictionaryValueConverter< TARGET_TYPE > Struct Template Reference

#include <TargetValueConvertersImpl.h>

+ Inheritance diagram for DictionaryValueConverter< TARGET_TYPE >:
+ Collaboration diagram for DictionaryValueConverter< TARGET_TYPE >:

Public Types

using ElementsDataColumnPtr = typename NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr
 
using ElementsBufferColumnPtr = std::unique_ptr< std::vector< int32_t >>
 
- Public Types inherited from NumericValueConverter< int64_t, TARGET_TYPE >
using ColumnDataPtr = std::unique_ptr< TARGET_TYPE, CheckedMallocDeleter< TARGET_TYPE >>
 
using ElementsBufferColumnPtr = ColumnDataPtr
 

Public Member Functions

 DictionaryValueConverter (const Catalog_Namespace::Catalog &cat, int32_t sourceDictId, const ColumnDescriptor *targetDescriptor, size_t num_rows, TARGET_TYPE nullValue, int64_t nullCheckValue, bool doNullCheck, StringDictionaryProxy *literals_dict, StringDictionaryProxy *source_dict_proxy)
 
 ~DictionaryValueConverter () override
 
ElementsBufferColumnPtr allocateColumnarBuffer (size_t num_rows)
 
void convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnBuffer, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const TargetValue *value) override
 
int32_t convertTransientStringIdToPermanentId (int32_t &transient_string_id)
 
NumericValueConverter< int64_t,
TARGET_TYPE >::ColumnDataPtr 
processBuffer (ElementsBufferColumnPtr buffer)
 
void processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)
 
void finalizeDataBlocksForInsertData () override
 
void addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override
 
- Public Member Functions inherited from NumericValueConverter< int64_t, TARGET_TYPE >
 NumericValueConverter (const ColumnDescriptor *cd, size_t num_rows, TARGET_TYPE nullValue, int64_tnullCheckValue, bool doNullCheck)
 
 ~NumericValueConverter () override
 
void allocateColumnarData (size_t num_rows) override
 
ElementsBufferColumnPtr allocateColumnarBuffer (size_t num_rows)
 
void convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnData, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const TargetValue *value) override
 
void processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)
 
void addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override
 
- Public Member Functions inherited from TargetValueConverter
 TargetValueConverter (const ColumnDescriptor *cd)
 
virtual ~TargetValueConverter ()
 

Public Attributes

ElementsBufferColumnPtr column_buffer_
 
const DictDescriptortarget_dict_desc_
 
const DictDescriptorsource_dict_desc_
 
StringDictionaryProxysource_dict_proxy_
 
const int32_t buffer_null_sentinal_ = std::numeric_limits<int32_t>::min()
 
const StringDictionaryProxyliterals_dict_
 
std::unordered_map< int32_t,
int32_t > 
literals_lookup_
 
bool use_literals_
 
- Public Attributes inherited from NumericValueConverter< int64_t, TARGET_TYPE >
ColumnDataPtr column_data_
 
TARGET_TYPE null_value_
 
int64_t null_check_value_
 
bool do_null_check_
 
boost_variant_accessor< int64_t > SOURCE_TYPE_ACCESSOR
 
- Public Attributes inherited from TargetValueConverter
const ColumnDescriptorcolumn_descriptor_
 
boost_variant_accessor
< ScalarTargetValue
SCALAR_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor
< ArrayTargetValue
ARRAY_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor
< GeoTargetValue
GEO_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor
< NullableString
NULLABLE_STRING_ACCESSOR
 
boost_variant_accessor
< std::string > 
STRING_ACCESSOR
 

Detailed Description

template<typename TARGET_TYPE>
struct DictionaryValueConverter< TARGET_TYPE >

Definition at line 120 of file TargetValueConvertersImpl.h.

Member Typedef Documentation

template<typename TARGET_TYPE >
using DictionaryValueConverter< TARGET_TYPE >::ElementsBufferColumnPtr = std::unique_ptr<std::vector<int32_t>>

Definition at line 124 of file TargetValueConvertersImpl.h.

template<typename TARGET_TYPE >
using DictionaryValueConverter< TARGET_TYPE >::ElementsDataColumnPtr = typename NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr

Definition at line 122 of file TargetValueConvertersImpl.h.

Constructor & Destructor Documentation

template<typename TARGET_TYPE >
DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter ( const Catalog_Namespace::Catalog cat,
int32_t  sourceDictId,
const ColumnDescriptor targetDescriptor,
size_t  num_rows,
TARGET_TYPE  nullValue,
int64_t  nullCheckValue,
bool  doNullCheck,
StringDictionaryProxy literals_dict,
StringDictionaryProxy source_dict_proxy 
)
inline

Definition at line 140 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::allocateColumnarBuffer(), DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, CHECK(), DictionaryValueConverter< TARGET_TYPE >::column_buffer_, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), Catalog_Namespace::Catalog::getMetadataForDict(), StringDictionaryProxy::getTransientMapping(), DictionaryValueConverter< TARGET_TYPE >::literals_dict_, DictionaryValueConverter< TARGET_TYPE >::literals_lookup_, DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

150  num_rows,
151  nullValue,
152  nullCheckValue,
153  doNullCheck) {
154  literals_dict_ = literals_dict;
156  cat.getMetadataForDict(targetDescriptor->columnType.get_comp_param(), true);
157 
158  source_dict_desc_ = nullptr;
159  source_dict_proxy_ = source_dict_proxy;
160 
161  use_literals_ = 0 == sourceDictId;
162  if (!use_literals_) {
163  source_dict_desc_ = cat.getMetadataForDict(std::abs(sourceDictId), true);
165  } else {
166  CHECK(literals_dict);
167 
168  for (auto& entry : literals_dict->getTransientMapping()) {
169  auto newId = target_dict_desc_->stringDict->getOrAdd(entry.second);
170  literals_lookup_[entry.first] = newId;
171  }
172 
174  }
175 
177 
178  if (num_rows) {
180  }
181  }
const int8_t const int64_t * num_rows
const DictDescriptor * source_dict_desc_
ElementsBufferColumnPtr column_buffer_
ElementsBufferColumnPtr allocateColumnarBuffer(size_t num_rows)
const DictDescriptor * target_dict_desc_
const StringDictionaryProxy * literals_dict_
std::shared_ptr< StringDictionary > stringDict
CHECK(cgen_state)
const std::map< int32_t, std::string > getTransientMapping() const
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1350
std::unordered_map< int32_t, int32_t > literals_lookup_
StringDictionaryProxy * source_dict_proxy_

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
DictionaryValueConverter< TARGET_TYPE >::~DictionaryValueConverter ( )
inlineoverride

Definition at line 183 of file TargetValueConvertersImpl.h.

183 {}

Member Function Documentation

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::addDataBlocksToInsertData ( Fragmenter_Namespace::InsertData insertData)
inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 346 of file TargetValueConvertersImpl.h.

References NumericValueConverter< int64_t, TARGET_TYPE >::column_data_, TargetValueConverter::column_descriptor_, ColumnDescriptor::columnId, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData(), and DataBlockPtr::numbersPtr.

346  {
348  DataBlockPtr dataBlock;
349  dataBlock.numbersPtr = reinterpret_cast<int8_t*>(this->column_data_.get());
350  insertData.data.push_back(dataBlock);
351  insertData.columnIds.push_back(this->column_descriptor_->columnId);
352  }
const ColumnDescriptor * column_descriptor_
void finalizeDataBlocksForInsertData() override
std::vector< DataBlockPtr > data
the number of rows being inserted
Definition: Fragmenter.h:64
int8_t * numbersPtr
Definition: sqltypes.h:140
std::vector< int > columnIds
identifies the table into which the data is being inserted
Definition: Fragmenter.h:62

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
ElementsBufferColumnPtr DictionaryValueConverter< TARGET_TYPE >::allocateColumnarBuffer ( size_t  num_rows)
inline

Definition at line 185 of file TargetValueConvertersImpl.h.

References CHECK(), and num_rows.

Referenced by DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter().

185  {
186  CHECK(num_rows > 0);
187  return std::make_unique<std::vector<int32_t>>(num_rows);
188  }
const int8_t const int64_t * num_rows
CHECK(cgen_state)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertElementToColumnarFormat ( size_t  row,
typename ElementsBufferColumnPtr::pointer  columnBuffer,
const ScalarTargetValue scalarValue 
)
inline

Definition at line 190 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, NumericValueConverter< int64_t, TARGET_TYPE >::do_null_check_, NumericValueConverter< int64_t, TARGET_TYPE >::null_check_value_, and NumericValueConverter< int64_t, TARGET_TYPE >::SOURCE_TYPE_ACCESSOR.

Referenced by DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat().

193  {
194  auto mapd_p = checked_get<int64_t>(row, scalarValue, this->SOURCE_TYPE_ACCESSOR);
195  auto val = *mapd_p;
196 
197  if (this->do_null_check_ && this->null_check_value_ == val) {
198  (*columnBuffer)[row] = this->buffer_null_sentinal_;
199  } else {
200  (*columnBuffer)[row] = (int32_t)val;
201  }
202  }
boost_variant_accessor< int64_t > SOURCE_TYPE_ACCESSOR

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat ( size_t  row,
const ScalarTargetValue scalarValue 
)
inline

Definition at line 204 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::column_buffer_, and DictionaryValueConverter< TARGET_TYPE >::convertElementToColumnarFormat().

Referenced by DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat().

204  {
205  convertElementToColumnarFormat(row, this->column_buffer_.get(), scalarValue);
206  }
ElementsBufferColumnPtr column_buffer_
void convertElementToColumnarFormat(size_t row, typename ElementsBufferColumnPtr::pointer columnBuffer, const ScalarTargetValue *scalarValue)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat ( size_t  row,
const TargetValue value 
)
inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 208 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat(), and TargetValueConverter::SCALAR_TARGET_VALUE_ACCESSOR.

208  {
209  auto scalarValue =
210  checked_get<ScalarTargetValue>(row, value, this->SCALAR_TARGET_VALUE_ACCESSOR);
211 
212  convertToColumnarFormat(row, scalarValue);
213  }
void convertToColumnarFormat(size_t row, const ScalarTargetValue *scalarValue)
boost_variant_accessor< ScalarTargetValue > SCALAR_TARGET_VALUE_ACCESSOR

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
int32_t DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId ( int32_t &  transient_string_id)
inline

Definition at line 215 of file TargetValueConvertersImpl.h.

References StringDictionaryProxy::getOrAdd(), StringDictionaryProxy::getString(), and DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_.

Referenced by DictionaryValueConverter< TARGET_TYPE >::processBuffer().

215  {
216  if (source_dict_proxy_) {
217  auto str = source_dict_proxy_->getString(transient_string_id);
218  return source_dict_proxy_->getOrAdd(str);
219  } else {
220  throw std::runtime_error("Unexpected negative source ID");
221  }
222  }
std::string getString(int32_t string_id) const
int32_t getOrAdd(const std::string &str) noexcept
StringDictionaryProxy * source_dict_proxy_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData ( )
inlineoverridevirtual

Reimplemented from TargetValueConverter.

Definition at line 339 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::column_buffer_, NumericValueConverter< int64_t, TARGET_TYPE >::column_data_, and DictionaryValueConverter< TARGET_TYPE >::processBuffer().

Referenced by DictionaryValueConverter< TARGET_TYPE >::addDataBlocksToInsertData().

339  {
340  if (column_buffer_) {
341  this->column_data_ = processBuffer(std::move(column_buffer_));
342  column_buffer_ = nullptr;
343  }
344  }
ElementsBufferColumnPtr column_buffer_
NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr processBuffer(ElementsBufferColumnPtr buffer)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::processArrayBuffer ( std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &  array_buffer,
std::unique_ptr< std::vector< ArrayDatum >>::pointer  arrayData 
)
inline

Definition at line 292 of file TargetValueConvertersImpl.h.

References anonymous_namespace{TypedDataAccessors.h}::is_null(), StringDictionary::populate_string_array_ids(), DictionaryValueConverter< TARGET_TYPE >::processBuffer(), DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

295  {
296  if (use_literals_) {
297  for (size_t row = 0; row < array_buffer->size(); row++) {
298  auto& element = (array_buffer->at(row));
299  bool is_null = false;
300  if (element.second) {
302  processBuffer(std::move(element.second));
303  int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(data.release());
304  (*arrayData)[row] =
305  ArrayDatum(element.first * sizeof(TARGET_TYPE), arrayDataPtr, is_null);
306  }
307  }
308  } else {
309  std::vector<std::vector<int32_t>> srcArrayIds(array_buffer->size());
310  std::vector<std::vector<int32_t>> destArrayIds(0);
311 
312  for (size_t row = 0; row < array_buffer->size(); row++) {
313  auto& element = (array_buffer->at(row));
314  if (element.second) {
315  srcArrayIds[row] = *(element.second.get());
316  }
317  }
318 
321  srcArrayIds,
323 
324  for (size_t row = 0; row < array_buffer->size(); row++) {
325  auto& element = (array_buffer->at(row));
326  bool is_null = false;
327  if (element.second) {
328  *(element.second.get()) = destArrayIds[row];
329  int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(&(element.second->at(0)));
330  (*arrayData)[row] = ArrayDatum(element.first * sizeof(TARGET_TYPE),
331  arrayDataPtr,
332  is_null,
333  DoNothingDeleter());
334  }
335  }
336  }
337  }
const DictDescriptor * source_dict_desc_
const DictDescriptor * target_dict_desc_
std::shared_ptr< StringDictionary > stringDict
bool is_null(const T &v, const SQLTypeInfo &t)
static void populate_string_array_ids(std::vector< std::vector< int32_t >> &dest_array_ids, StringDictionary *dest_dict, const std::vector< std::vector< int32_t >> &source_array_ids, const StringDictionary *source_dict)
NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr processBuffer(ElementsBufferColumnPtr buffer)
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:122

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr DictionaryValueConverter< TARGET_TYPE >::processBuffer ( ElementsBufferColumnPtr  buffer)
inline

Definition at line 224 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, CHECK(), checked_malloc(), DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId(), DictDescriptor::dictRef, StringDictionaryProxy::getTransientMapping(), DictionaryValueConverter< TARGET_TYPE >::literals_lookup_, NumericValueConverter< int64_t, TARGET_TYPE >::null_value_, StringDictionary::populate_string_ids(), DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

Referenced by DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData(), and DictionaryValueConverter< TARGET_TYPE >::processArrayBuffer().

225  {
228  reinterpret_cast<TARGET_TYPE*>(
229  checked_malloc(buffer->size() * sizeof(TARGET_TYPE))));
230 
231  std::vector<int32_t>* bufferPtr =
232  reinterpret_cast<std::vector<int32_t>*>(buffer.get());
233  TARGET_TYPE* columnDataPtr = reinterpret_cast<TARGET_TYPE*>(data.get());
234  if (use_literals_) {
235  for (size_t i = 0; i < bufferPtr->size(); i++) {
236  auto id = literals_lookup_[(*bufferPtr)[i]];
237  if (id == buffer_null_sentinal_) {
238  columnDataPtr[i] = this->null_value_;
239  } else {
240  columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
241  }
242  }
243  } else {
245  // special case, where source and target dict are the same
246  // mostly in update
247  for (size_t i = 0; i < bufferPtr->size(); i++) {
248  auto src_id = (*bufferPtr)[i];
249  if (src_id == buffer_null_sentinal_) {
250  columnDataPtr[i] = this->null_value_;
251  } else if (src_id < 0) {
252  columnDataPtr[i] = convertTransientStringIdToPermanentId(src_id);
253  } else {
254  columnDataPtr[i] = static_cast<TARGET_TYPE>(src_id);
255  }
256  }
257 
258  } else {
259  std::vector<int32_t> dest_ids;
260  dest_ids.resize(bufferPtr->size());
261 
262  if (source_dict_proxy_) {
264  dest_ids,
266  *bufferPtr,
269  } else {
272  *bufferPtr,
274  }
275 
276  // fixup NULL sentinel
277  for (size_t i = 0; i < dest_ids.size(); i++) {
278  auto id = dest_ids[i];
279  if (id == buffer_null_sentinal_) {
280  columnDataPtr[i] = this->null_value_;
281  } else {
282  CHECK(std::numeric_limits<TARGET_TYPE>::max() >= id);
283  columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
284  }
285  }
286  }
287  }
288 
289  return data;
290  }
std::unique_ptr< TARGET_TYPE, CheckedMallocDeleter< TARGET_TYPE >> ColumnDataPtr
const DictDescriptor * source_dict_desc_
static void populate_string_ids(std::vector< int32_t > &dest_ids, StringDictionary *dest_dict, const std::vector< int32_t > &source_ids, const StringDictionary *source_dict, const std::map< int32_t, std::string > transient_mapping={})
Populates provided dest_ids vector with string ids corresponding to given source strings.
const DictDescriptor * target_dict_desc_
std::shared_ptr< StringDictionary > stringDict
CHECK(cgen_state)
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
const std::map< int32_t, std::string > getTransientMapping() const
int32_t convertTransientStringIdToPermanentId(int32_t &transient_string_id)
std::unordered_map< int32_t, int32_t > literals_lookup_
StringDictionaryProxy * source_dict_proxy_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

template<typename TARGET_TYPE >
const int32_t DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_ = std::numeric_limits<int32_t>::min()
template<typename TARGET_TYPE >
const StringDictionaryProxy* DictionaryValueConverter< TARGET_TYPE >::literals_dict_
template<typename TARGET_TYPE >
std::unordered_map<int32_t, int32_t> DictionaryValueConverter< TARGET_TYPE >::literals_lookup_

The documentation for this struct was generated from the following file: