OmniSciDB  29e35f4d58
DictionaryValueConverter< TARGET_TYPE > Struct Template Reference

#include <TargetValueConvertersImpl.h>

+ Inheritance diagram for DictionaryValueConverter< TARGET_TYPE >:
+ Collaboration diagram for DictionaryValueConverter< TARGET_TYPE >:

Public Types

using ElementsDataColumnPtr = typename NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr
 
using ElementsBufferColumnPtr = std::unique_ptr< std::vector< int32_t > >
 
- Public Types inherited from NumericValueConverter< int64_t, TARGET_TYPE >
using ColumnDataPtr = std::unique_ptr< TARGET_TYPE, CheckedMallocDeleter< TARGET_TYPE > >
 
using ElementsBufferColumnPtr = ColumnDataPtr
 

Public Member Functions

 DictionaryValueConverter (const Catalog_Namespace::Catalog &cat, int32_t sourceDictId, const ColumnDescriptor *targetDescriptor, size_t num_rows, TARGET_TYPE nullValue, int64_t nullCheckValue, bool doNullCheck, StringDictionaryProxy *literals_dict, StringDictionaryProxy *source_dict_proxy)
 
 ~DictionaryValueConverter () override
 
bool allowFixedNullArray ()
 
ElementsBufferColumnPtr allocateColumnarBuffer (size_t num_rows)
 
void convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnBuffer, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const TargetValue *value) override
 
int32_t convertTransientStringIdToPermanentId (int32_t &transient_string_id)
 
NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr processBuffer (ElementsBufferColumnPtr buffer)
 
void processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)
 
void finalizeDataBlocksForInsertData () override
 
void addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override
 
- Public Member Functions inherited from NumericValueConverter< int64_t, TARGET_TYPE >
 NumericValueConverter (const ColumnDescriptor *cd, size_t num_rows, TARGET_TYPE nullValue, int64_t nullCheckValue, bool doNullCheck)
 
 ~NumericValueConverter () override
 
bool allowFixedNullArray ()
 
void populateFixedArrayNullSentinel (size_t num_rows)
 
void allocateColumnarData (size_t num_rows) override
 
ElementsBufferColumnPtr allocateColumnarBuffer (size_t num_rows)
 
void convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnData, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const TargetValue *value) override
 
void processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)
 
void addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override
 
- Public Member Functions inherited from TargetValueConverter
 TargetValueConverter (const ColumnDescriptor *cd)
 
virtual ~TargetValueConverter ()
 

Public Attributes

ElementsBufferColumnPtr column_buffer_
 
const DictDescriptortarget_dict_desc_
 
const DictDescriptorsource_dict_desc_
 
StringDictionaryProxysource_dict_proxy_
 
const int32_t buffer_null_sentinal_ = std::numeric_limits<int32_t>::min()
 
const StringDictionaryProxyliterals_dict_
 
std::unordered_map< int32_t, int32_t > literals_lookup_
 
bool use_literals_
 
- Public Attributes inherited from NumericValueConverter< int64_t, TARGET_TYPE >
ColumnDataPtr column_data_
 
TARGET_TYPE null_value_
 
int64_t null_check_value_
 
bool do_null_check_
 
TARGET_TYPE fixed_array_null_value_
 
boost_variant_accessor< int64_t > SOURCE_TYPE_ACCESSOR
 
- Public Attributes inherited from TargetValueConverter
const ColumnDescriptorcolumn_descriptor_
 
boost_variant_accessor< ScalarTargetValueSCALAR_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor< ArrayTargetValueARRAY_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor< GeoTargetValueGEO_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor< NullableStringNULLABLE_STRING_ACCESSOR
 
boost_variant_accessor< std::string > STRING_ACCESSOR
 

Detailed Description

template<typename TARGET_TYPE>
struct DictionaryValueConverter< TARGET_TYPE >

Definition at line 139 of file TargetValueConvertersImpl.h.

Member Typedef Documentation

◆ ElementsBufferColumnPtr

template<typename TARGET_TYPE >
using DictionaryValueConverter< TARGET_TYPE >::ElementsBufferColumnPtr = std::unique_ptr<std::vector<int32_t> >

Definition at line 143 of file TargetValueConvertersImpl.h.

◆ ElementsDataColumnPtr

template<typename TARGET_TYPE >
using DictionaryValueConverter< TARGET_TYPE >::ElementsDataColumnPtr = typename NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr

Definition at line 141 of file TargetValueConvertersImpl.h.

Constructor & Destructor Documentation

◆ DictionaryValueConverter()

template<typename TARGET_TYPE >
DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter ( const Catalog_Namespace::Catalog cat,
int32_t  sourceDictId,
const ColumnDescriptor targetDescriptor,
size_t  num_rows,
TARGET_TYPE  nullValue,
int64_t  nullCheckValue,
bool  doNullCheck,
StringDictionaryProxy literals_dict,
StringDictionaryProxy source_dict_proxy 
)
inline

Definition at line 159 of file TargetValueConvertersImpl.h.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), Catalog_Namespace::Catalog::getMetadataForDict(), StringDictionaryProxy::getTransientMapping(), and DictDescriptor::stringDict.

169  num_rows,
170  nullValue,
171  nullCheckValue,
172  doNullCheck) {
173  literals_dict_ = literals_dict;
175  cat.getMetadataForDict(targetDescriptor->columnType.get_comp_param(), true);
176 
177  source_dict_desc_ = nullptr;
178  source_dict_proxy_ = source_dict_proxy;
179 
180  use_literals_ = 0 == sourceDictId;
181  if (!use_literals_) {
182  source_dict_desc_ = cat.getMetadataForDict(std::abs(sourceDictId), true);
184  } else {
185  if (literals_dict) {
186  for (auto& entry : literals_dict->getTransientMapping()) {
187  auto newId = target_dict_desc_->stringDict->getOrAdd(entry.second);
188  literals_lookup_[entry.first] = newId;
189  }
190  }
191 
193  }
194 
196 
197  if (num_rows) {
199  }
200  }
const int8_t const int64_t * num_rows
const DictDescriptor * source_dict_desc_
ElementsBufferColumnPtr column_buffer_
ElementsBufferColumnPtr allocateColumnarBuffer(size_t num_rows)
const DictDescriptor * target_dict_desc_
const StringDictionaryProxy * literals_dict_
std::shared_ptr< StringDictionary > stringDict
const std::map< int32_t, std::string > getTransientMapping() const
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1377
std::unordered_map< int32_t, int32_t > literals_lookup_
#define CHECK(condition)
Definition: Logger.h:193
StringDictionaryProxy * source_dict_proxy_
+ Here is the call graph for this function:

◆ ~DictionaryValueConverter()

template<typename TARGET_TYPE >
DictionaryValueConverter< TARGET_TYPE >::~DictionaryValueConverter ( )
inlineoverride

Definition at line 202 of file TargetValueConvertersImpl.h.

202 {}

Member Function Documentation

◆ addDataBlocksToInsertData()

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::addDataBlocksToInsertData ( Fragmenter_Namespace::InsertData insertData)
inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 367 of file TargetValueConvertersImpl.h.

References Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, and DataBlockPtr::numbersPtr.

367  {
369  DataBlockPtr dataBlock;
370  dataBlock.numbersPtr = reinterpret_cast<int8_t*>(this->column_data_.get());
371  insertData.data.push_back(dataBlock);
372  insertData.columnIds.push_back(this->column_descriptor_->columnId);
373  }
const ColumnDescriptor * column_descriptor_
void finalizeDataBlocksForInsertData() override
std::vector< DataBlockPtr > data
the number of rows being inserted
Definition: Fragmenter.h:64
int8_t * numbersPtr
Definition: sqltypes.h:140
std::vector< int > columnIds
identifies the table into which the data is being inserted
Definition: Fragmenter.h:62

◆ allocateColumnarBuffer()

template<typename TARGET_TYPE >
ElementsBufferColumnPtr DictionaryValueConverter< TARGET_TYPE >::allocateColumnarBuffer ( size_t  num_rows)
inline

Definition at line 206 of file TargetValueConvertersImpl.h.

References CHECK, and num_rows.

206  {
207  CHECK(num_rows > 0);
208  return std::make_unique<std::vector<int32_t>>(num_rows);
209  }
const int8_t const int64_t * num_rows
#define CHECK(condition)
Definition: Logger.h:193

◆ allowFixedNullArray()

template<typename TARGET_TYPE >
bool DictionaryValueConverter< TARGET_TYPE >::allowFixedNullArray ( )
inline

Definition at line 204 of file TargetValueConvertersImpl.h.

204 { return false; }

◆ convertElementToColumnarFormat()

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertElementToColumnarFormat ( size_t  row,
typename ElementsBufferColumnPtr::pointer  columnBuffer,
const ScalarTargetValue scalarValue 
)
inline

Definition at line 211 of file TargetValueConvertersImpl.h.

214  {
215  auto mapd_p = checked_get<int64_t>(row, scalarValue, this->SOURCE_TYPE_ACCESSOR);
216  auto val = *mapd_p;
217 
218  if (this->do_null_check_ && this->null_check_value_ == val) {
219  (*columnBuffer)[row] = this->buffer_null_sentinal_;
220  } else {
221  (*columnBuffer)[row] = (int32_t)val;
222  }
223  }
boost_variant_accessor< int64_t > SOURCE_TYPE_ACCESSOR

◆ convertToColumnarFormat() [1/2]

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat ( size_t  row,
const ScalarTargetValue scalarValue 
)
inline

Definition at line 225 of file TargetValueConvertersImpl.h.

225  {
226  convertElementToColumnarFormat(row, this->column_buffer_.get(), scalarValue);
227  }
ElementsBufferColumnPtr column_buffer_
void convertElementToColumnarFormat(size_t row, typename ElementsBufferColumnPtr::pointer columnBuffer, const ScalarTargetValue *scalarValue)

◆ convertToColumnarFormat() [2/2]

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat ( size_t  row,
const TargetValue value 
)
inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 229 of file TargetValueConvertersImpl.h.

229  {
230  auto scalarValue =
231  checked_get<ScalarTargetValue>(row, value, this->SCALAR_TARGET_VALUE_ACCESSOR);
232 
233  convertToColumnarFormat(row, scalarValue);
234  }
void convertToColumnarFormat(size_t row, const ScalarTargetValue *scalarValue)
boost_variant_accessor< ScalarTargetValue > SCALAR_TARGET_VALUE_ACCESSOR

◆ convertTransientStringIdToPermanentId()

template<typename TARGET_TYPE >
int32_t DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId ( int32_t &  transient_string_id)
inline

Definition at line 236 of file TargetValueConvertersImpl.h.

References StringDictionaryProxy::getOrAdd(), and StringDictionaryProxy::getString().

236  {
237  if (source_dict_proxy_) {
238  auto str = source_dict_proxy_->getString(transient_string_id);
239  return source_dict_proxy_->getOrAdd(str);
240  } else {
241  throw std::runtime_error("Unexpected negative source ID");
242  }
243  }
std::string getString(int32_t string_id) const
int32_t getOrAdd(const std::string &str) noexcept
StringDictionaryProxy * source_dict_proxy_
+ Here is the call graph for this function:

◆ finalizeDataBlocksForInsertData()

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData ( )
inlineoverridevirtual

Reimplemented from TargetValueConverter.

Definition at line 360 of file TargetValueConvertersImpl.h.

360  {
361  if (column_buffer_) {
362  this->column_data_ = processBuffer(std::move(column_buffer_));
363  column_buffer_ = nullptr;
364  }
365  }
ElementsBufferColumnPtr column_buffer_
NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr processBuffer(ElementsBufferColumnPtr buffer)

◆ processArrayBuffer()

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::processArrayBuffer ( std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &  array_buffer,
std::unique_ptr< std::vector< ArrayDatum >>::pointer  arrayData 
)
inline

Definition at line 313 of file TargetValueConvertersImpl.h.

References anonymous_namespace{TypedDataAccessors.h}::is_null(), StringDictionary::populate_string_array_ids(), and DictDescriptor::stringDict.

316  {
317  if (use_literals_) {
318  for (size_t row = 0; row < array_buffer->size(); row++) {
319  auto& element = (array_buffer->at(row));
320  bool is_null = false;
321  if (element.second) {
323  processBuffer(std::move(element.second));
324  int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(data.release());
325  (*arrayData)[row] =
326  ArrayDatum(element.first * sizeof(TARGET_TYPE), arrayDataPtr, is_null);
327  }
328  }
329  } else {
330  std::vector<std::vector<int32_t>> srcArrayIds(array_buffer->size());
331  std::vector<std::vector<int32_t>> destArrayIds(0);
332 
333  for (size_t row = 0; row < array_buffer->size(); row++) {
334  auto& element = (array_buffer->at(row));
335  if (element.second) {
336  srcArrayIds[row] = *(element.second.get());
337  }
338  }
339 
342  srcArrayIds,
344 
345  for (size_t row = 0; row < array_buffer->size(); row++) {
346  auto& element = (array_buffer->at(row));
347  bool is_null = false;
348  if (element.second) {
349  *(element.second.get()) = destArrayIds[row];
350  int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(&(element.second->at(0)));
351  (*arrayData)[row] = ArrayDatum(element.first * sizeof(TARGET_TYPE),
352  arrayDataPtr,
353  is_null,
354  DoNothingDeleter());
355  }
356  }
357  }
358  }
const DictDescriptor * source_dict_desc_
const DictDescriptor * target_dict_desc_
std::shared_ptr< StringDictionary > stringDict
bool is_null(const T &v, const SQLTypeInfo &t)
static void populate_string_array_ids(std::vector< std::vector< int32_t >> &dest_array_ids, StringDictionary *dest_dict, const std::vector< std::vector< int32_t >> &source_array_ids, const StringDictionary *source_dict)
NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr processBuffer(ElementsBufferColumnPtr buffer)
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:122
+ Here is the call graph for this function:

◆ processBuffer()

template<typename TARGET_TYPE >
NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr DictionaryValueConverter< TARGET_TYPE >::processBuffer ( ElementsBufferColumnPtr  buffer)
inline

Definition at line 245 of file TargetValueConvertersImpl.h.

References CHECK, checked_malloc(), DictDescriptor::dictRef, StringDictionaryProxy::getTransientMapping(), StringDictionary::populate_string_ids(), and DictDescriptor::stringDict.

246  {
249  reinterpret_cast<TARGET_TYPE*>(
250  checked_malloc(buffer->size() * sizeof(TARGET_TYPE))));
251 
252  std::vector<int32_t>* bufferPtr =
253  reinterpret_cast<std::vector<int32_t>*>(buffer.get());
254  TARGET_TYPE* columnDataPtr = reinterpret_cast<TARGET_TYPE*>(data.get());
255  if (use_literals_) {
256  for (size_t i = 0; i < bufferPtr->size(); i++) {
257  auto id = literals_lookup_[(*bufferPtr)[i]];
258  if (id == buffer_null_sentinal_) {
259  columnDataPtr[i] = this->null_value_;
260  } else {
261  columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
262  }
263  }
264  } else {
266  // special case, where source and target dict are the same
267  // mostly in update
268  for (size_t i = 0; i < bufferPtr->size(); i++) {
269  auto src_id = (*bufferPtr)[i];
270  if (src_id == buffer_null_sentinal_) {
271  columnDataPtr[i] = this->null_value_;
272  } else if (src_id < 0) {
273  columnDataPtr[i] = convertTransientStringIdToPermanentId(src_id);
274  } else {
275  columnDataPtr[i] = static_cast<TARGET_TYPE>(src_id);
276  }
277  }
278 
279  } else {
280  std::vector<int32_t> dest_ids;
281  dest_ids.resize(bufferPtr->size());
282 
283  if (source_dict_proxy_) {
285  dest_ids,
287  *bufferPtr,
290  } else {
293  *bufferPtr,
295  }
296 
297  // fixup NULL sentinel
298  for (size_t i = 0; i < dest_ids.size(); i++) {
299  auto id = dest_ids[i];
300  if (id == buffer_null_sentinal_) {
301  columnDataPtr[i] = this->null_value_;
302  } else {
303  CHECK(std::numeric_limits<TARGET_TYPE>::max() >= id);
304  columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
305  }
306  }
307  }
308  }
309 
310  return data;
311  }
const DictDescriptor * source_dict_desc_
static void populate_string_ids(std::vector< int32_t > &dest_ids, StringDictionary *dest_dict, const std::vector< int32_t > &source_ids, const StringDictionary *source_dict, const std::map< int32_t, std::string > transient_mapping={})
Populates provided dest_ids vector with string ids corresponding to given source strings.
const DictDescriptor * target_dict_desc_
std::shared_ptr< StringDictionary > stringDict
const std::map< int32_t, std::string > getTransientMapping() const
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
int32_t convertTransientStringIdToPermanentId(int32_t &transient_string_id)
std::unordered_map< int32_t, int32_t > literals_lookup_
#define CHECK(condition)
Definition: Logger.h:193
std::unique_ptr< TARGET_TYPE, CheckedMallocDeleter< TARGET_TYPE > > ColumnDataPtr
StringDictionaryProxy * source_dict_proxy_
+ Here is the call graph for this function:

Member Data Documentation

◆ buffer_null_sentinal_

template<typename TARGET_TYPE >
const int32_t DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_ = std::numeric_limits<int32_t>::min()

Definition at line 152 of file TargetValueConvertersImpl.h.

◆ column_buffer_

template<typename TARGET_TYPE >
ElementsBufferColumnPtr DictionaryValueConverter< TARGET_TYPE >::column_buffer_

Definition at line 145 of file TargetValueConvertersImpl.h.

◆ literals_dict_

template<typename TARGET_TYPE >
const StringDictionaryProxy* DictionaryValueConverter< TARGET_TYPE >::literals_dict_

Definition at line 154 of file TargetValueConvertersImpl.h.

◆ literals_lookup_

template<typename TARGET_TYPE >
std::unordered_map<int32_t, int32_t> DictionaryValueConverter< TARGET_TYPE >::literals_lookup_

Definition at line 156 of file TargetValueConvertersImpl.h.

◆ source_dict_desc_

template<typename TARGET_TYPE >
const DictDescriptor* DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_

Definition at line 148 of file TargetValueConvertersImpl.h.

◆ source_dict_proxy_

template<typename TARGET_TYPE >
StringDictionaryProxy* DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_

Definition at line 150 of file TargetValueConvertersImpl.h.

◆ target_dict_desc_

template<typename TARGET_TYPE >
const DictDescriptor* DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_

Definition at line 147 of file TargetValueConvertersImpl.h.

◆ use_literals_

template<typename TARGET_TYPE >
bool DictionaryValueConverter< TARGET_TYPE >::use_literals_

Definition at line 157 of file TargetValueConvertersImpl.h.


The documentation for this struct was generated from the following file: