OmniSciDB  0fdbebe030
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
DictionaryValueConverter< TARGET_TYPE > Struct Template Reference

#include <TargetValueConvertersImpl.h>

+ Inheritance diagram for DictionaryValueConverter< TARGET_TYPE >:
+ Collaboration diagram for DictionaryValueConverter< TARGET_TYPE >:

Public Types

using ElementsDataColumnPtr = typename NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr
 
using ElementsBufferColumnPtr = std::unique_ptr< std::vector< int32_t >>
 
- Public Types inherited from NumericValueConverter< int64_t, TARGET_TYPE >
using ColumnDataPtr = std::unique_ptr< TARGET_TYPE, CheckedMallocDeleter< TARGET_TYPE >>
 
using ElementsBufferColumnPtr = ColumnDataPtr
 

Public Member Functions

 DictionaryValueConverter (const Catalog_Namespace::Catalog &cat, int32_t sourceDictId, const ColumnDescriptor *targetDescriptor, size_t num_rows, TARGET_TYPE nullValue, int64_t nullCheckValue, bool doNullCheck, StringDictionaryProxy *literals_dict, StringDictionaryProxy *source_dict_proxy)
 
 ~DictionaryValueConverter () override
 
bool allowFixedNullArray ()
 
ElementsBufferColumnPtr allocateColumnarBuffer (size_t num_rows)
 
void convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnBuffer, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const TargetValue *value) override
 
int32_t convertTransientStringIdToPermanentId (int32_t &transient_string_id)
 
NumericValueConverter< int64_t,
TARGET_TYPE >::ColumnDataPtr 
processBuffer (ElementsBufferColumnPtr buffer)
 
void processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)
 
void finalizeDataBlocksForInsertData () override
 
void addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override
 
- Public Member Functions inherited from NumericValueConverter< int64_t, TARGET_TYPE >
 NumericValueConverter (const ColumnDescriptor *cd, size_t num_rows, TARGET_TYPE nullValue, int64_tnullCheckValue, bool doNullCheck)
 
 ~NumericValueConverter () override
 
bool allowFixedNullArray ()
 
void populateFixedArrayNullSentinel (size_t num_rows)
 
void allocateColumnarData (size_t num_rows) override
 
ElementsBufferColumnPtr allocateColumnarBuffer (size_t num_rows)
 
void convertElementToColumnarFormat (size_t row, typename ElementsBufferColumnPtr::pointer columnData, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const ScalarTargetValue *scalarValue)
 
void convertToColumnarFormat (size_t row, const TargetValue *value) override
 
void processArrayBuffer (std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &array_buffer, std::unique_ptr< std::vector< ArrayDatum >>::pointer arrayData)
 
void addDataBlocksToInsertData (Fragmenter_Namespace::InsertData &insertData) override
 
- Public Member Functions inherited from TargetValueConverter
 TargetValueConverter (const ColumnDescriptor *cd)
 
virtual ~TargetValueConverter ()
 

Public Attributes

ElementsBufferColumnPtr column_buffer_
 
const DictDescriptortarget_dict_desc_
 
const DictDescriptorsource_dict_desc_
 
StringDictionaryProxysource_dict_proxy_
 
const int32_t buffer_null_sentinal_ = std::numeric_limits<int32_t>::min()
 
const StringDictionaryProxyliterals_dict_
 
std::unordered_map< int32_t,
int32_t > 
literals_lookup_
 
bool use_literals_
 
- Public Attributes inherited from NumericValueConverter< int64_t, TARGET_TYPE >
ColumnDataPtr column_data_
 
TARGET_TYPE null_value_
 
int64_t null_check_value_
 
bool do_null_check_
 
TARGET_TYPE fixed_array_null_value_
 
boost_variant_accessor< int64_t > SOURCE_TYPE_ACCESSOR
 
- Public Attributes inherited from TargetValueConverter
const ColumnDescriptorcolumn_descriptor_
 
boost_variant_accessor
< ScalarTargetValue
SCALAR_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor
< ArrayTargetValue
ARRAY_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor
< GeoTargetValue
GEO_TARGET_VALUE_ACCESSOR
 
boost_variant_accessor
< NullableString
NULLABLE_STRING_ACCESSOR
 
boost_variant_accessor
< std::string > 
STRING_ACCESSOR
 

Detailed Description

template<typename TARGET_TYPE>
struct DictionaryValueConverter< TARGET_TYPE >

Definition at line 136 of file TargetValueConvertersImpl.h.

Member Typedef Documentation

template<typename TARGET_TYPE >
using DictionaryValueConverter< TARGET_TYPE >::ElementsBufferColumnPtr = std::unique_ptr<std::vector<int32_t>>

Definition at line 140 of file TargetValueConvertersImpl.h.

template<typename TARGET_TYPE >
using DictionaryValueConverter< TARGET_TYPE >::ElementsDataColumnPtr = typename NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr

Definition at line 138 of file TargetValueConvertersImpl.h.

Constructor & Destructor Documentation

template<typename TARGET_TYPE >
DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter ( const Catalog_Namespace::Catalog cat,
int32_t  sourceDictId,
const ColumnDescriptor targetDescriptor,
size_t  num_rows,
TARGET_TYPE  nullValue,
int64_t  nullCheckValue,
bool  doNullCheck,
StringDictionaryProxy literals_dict,
StringDictionaryProxy source_dict_proxy 
)
inline

Definition at line 156 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::allocateColumnarBuffer(), DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, CHECK(), DictionaryValueConverter< TARGET_TYPE >::column_buffer_, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::Catalog::getMetadataForDict(), StringDictionaryProxy::getTransientMapping(), DictionaryValueConverter< TARGET_TYPE >::literals_dict_, DictionaryValueConverter< TARGET_TYPE >::literals_lookup_, DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

166  num_rows,
167  nullValue,
168  nullCheckValue,
169  doNullCheck) {
170  literals_dict_ = literals_dict;
172  cat.getMetadataForDict(targetDescriptor->columnType.get_comp_param(), true);
173 
174  source_dict_desc_ = nullptr;
175  source_dict_proxy_ = source_dict_proxy;
176 
177  use_literals_ = 0 == sourceDictId;
178  if (!use_literals_) {
179  source_dict_desc_ = cat.getMetadataForDict(std::abs(sourceDictId), true);
181  } else {
182  if (literals_dict) {
183  for (auto& entry : literals_dict->getTransientMapping()) {
184  auto newId = target_dict_desc_->stringDict->getOrAdd(entry.second);
185  literals_lookup_[entry.first] = newId;
186  }
187  }
188 
190  }
191 
193 
194  if (num_rows) {
196  }
197  }
const int8_t const int64_t * num_rows
const DictDescriptor * source_dict_desc_
ElementsBufferColumnPtr column_buffer_
ElementsBufferColumnPtr allocateColumnarBuffer(size_t num_rows)
const DictDescriptor * target_dict_desc_
const StringDictionaryProxy * literals_dict_
std::shared_ptr< StringDictionary > stringDict
CHECK(cgen_state)
const std::map< int32_t, std::string > getTransientMapping() const
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1444
std::unordered_map< int32_t, int32_t > literals_lookup_
StringDictionaryProxy * source_dict_proxy_

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
DictionaryValueConverter< TARGET_TYPE >::~DictionaryValueConverter ( )
inlineoverride

Definition at line 199 of file TargetValueConvertersImpl.h.

199 {}

Member Function Documentation

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::addDataBlocksToInsertData ( Fragmenter_Namespace::InsertData insertData)
inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 364 of file TargetValueConvertersImpl.h.

References NumericValueConverter< int64_t, TARGET_TYPE >::column_data_, TargetValueConverter::column_descriptor_, ColumnDescriptor::columnId, Fragmenter_Namespace::InsertData::columnIds, Fragmenter_Namespace::InsertData::data, DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData(), and DataBlockPtr::numbersPtr.

364  {
366  DataBlockPtr dataBlock;
367  dataBlock.numbersPtr = reinterpret_cast<int8_t*>(this->column_data_.get());
368  insertData.data.push_back(dataBlock);
369  insertData.columnIds.push_back(this->column_descriptor_->columnId);
370  }
const ColumnDescriptor * column_descriptor_
void finalizeDataBlocksForInsertData() override
std::vector< DataBlockPtr > data
the number of rows being inserted
Definition: Fragmenter.h:64
int8_t * numbersPtr
Definition: sqltypes.h:138
std::vector< int > columnIds
identifies the table into which the data is being inserted
Definition: Fragmenter.h:62

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
ElementsBufferColumnPtr DictionaryValueConverter< TARGET_TYPE >::allocateColumnarBuffer ( size_t  num_rows)
inline

Definition at line 203 of file TargetValueConvertersImpl.h.

References CHECK(), and num_rows.

Referenced by DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter().

203  {
204  CHECK(num_rows > 0);
205  return std::make_unique<std::vector<int32_t>>(num_rows);
206  }
const int8_t const int64_t * num_rows
CHECK(cgen_state)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
bool DictionaryValueConverter< TARGET_TYPE >::allowFixedNullArray ( )
inline

Definition at line 201 of file TargetValueConvertersImpl.h.

201 { return false; }
template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertElementToColumnarFormat ( size_t  row,
typename ElementsBufferColumnPtr::pointer  columnBuffer,
const ScalarTargetValue scalarValue 
)
inline

Definition at line 208 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, NumericValueConverter< int64_t, TARGET_TYPE >::do_null_check_, NumericValueConverter< int64_t, TARGET_TYPE >::null_check_value_, and NumericValueConverter< int64_t, TARGET_TYPE >::SOURCE_TYPE_ACCESSOR.

Referenced by DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat().

211  {
212  auto mapd_p = checked_get<int64_t>(row, scalarValue, this->SOURCE_TYPE_ACCESSOR);
213  auto val = *mapd_p;
214 
215  if (this->do_null_check_ && this->null_check_value_ == val) {
216  (*columnBuffer)[row] = this->buffer_null_sentinal_;
217  } else {
218  (*columnBuffer)[row] = (int32_t)val;
219  }
220  }
boost_variant_accessor< int64_t > SOURCE_TYPE_ACCESSOR

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat ( size_t  row,
const ScalarTargetValue scalarValue 
)
inline

Definition at line 222 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::column_buffer_, and DictionaryValueConverter< TARGET_TYPE >::convertElementToColumnarFormat().

Referenced by DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat().

222  {
223  convertElementToColumnarFormat(row, this->column_buffer_.get(), scalarValue);
224  }
ElementsBufferColumnPtr column_buffer_
void convertElementToColumnarFormat(size_t row, typename ElementsBufferColumnPtr::pointer columnBuffer, const ScalarTargetValue *scalarValue)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat ( size_t  row,
const TargetValue value 
)
inlineoverridevirtual

Implements TargetValueConverter.

Definition at line 226 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::convertToColumnarFormat(), and TargetValueConverter::SCALAR_TARGET_VALUE_ACCESSOR.

226  {
227  auto scalarValue =
228  checked_get<ScalarTargetValue>(row, value, this->SCALAR_TARGET_VALUE_ACCESSOR);
229 
230  convertToColumnarFormat(row, scalarValue);
231  }
void convertToColumnarFormat(size_t row, const ScalarTargetValue *scalarValue)
boost_variant_accessor< ScalarTargetValue > SCALAR_TARGET_VALUE_ACCESSOR

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
int32_t DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId ( int32_t &  transient_string_id)
inline

Definition at line 233 of file TargetValueConvertersImpl.h.

References StringDictionaryProxy::getOrAdd(), StringDictionaryProxy::getString(), and DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_.

Referenced by DictionaryValueConverter< TARGET_TYPE >::processBuffer().

233  {
234  if (source_dict_proxy_) {
235  auto str = source_dict_proxy_->getString(transient_string_id);
236  return source_dict_proxy_->getOrAdd(str);
237  } else {
238  throw std::runtime_error("Unexpected negative source ID");
239  }
240  }
std::string getString(int32_t string_id) const
int32_t getOrAdd(const std::string &str) noexcept
StringDictionaryProxy * source_dict_proxy_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData ( )
inlineoverridevirtual

Reimplemented from TargetValueConverter.

Definition at line 357 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::column_buffer_, NumericValueConverter< int64_t, TARGET_TYPE >::column_data_, and DictionaryValueConverter< TARGET_TYPE >::processBuffer().

Referenced by DictionaryValueConverter< TARGET_TYPE >::addDataBlocksToInsertData().

357  {
358  if (column_buffer_) {
359  this->column_data_ = processBuffer(std::move(column_buffer_));
360  column_buffer_ = nullptr;
361  }
362  }
ElementsBufferColumnPtr column_buffer_
NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr processBuffer(ElementsBufferColumnPtr buffer)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename TARGET_TYPE >
void DictionaryValueConverter< TARGET_TYPE >::processArrayBuffer ( std::unique_ptr< std::vector< std::pair< size_t, ElementsBufferColumnPtr >>> &  array_buffer,
std::unique_ptr< std::vector< ArrayDatum >>::pointer  arrayData 
)
inline

Definition at line 310 of file TargetValueConvertersImpl.h.

References anonymous_namespace{TypedDataAccessors.h}::is_null(), StringDictionary::populate_string_array_ids(), DictionaryValueConverter< TARGET_TYPE >::processBuffer(), DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

313  {
314  if (use_literals_) {
315  for (size_t row = 0; row < array_buffer->size(); row++) {
316  auto& element = (array_buffer->at(row));
317  bool is_null = false;
318  if (element.second) {
320  processBuffer(std::move(element.second));
321  int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(data.release());
322  (*arrayData)[row] =
323  ArrayDatum(element.first * sizeof(TARGET_TYPE), arrayDataPtr, is_null);
324  }
325  }
326  } else {
327  std::vector<std::vector<int32_t>> srcArrayIds(array_buffer->size());
328  std::vector<std::vector<int32_t>> destArrayIds(0);
329 
330  for (size_t row = 0; row < array_buffer->size(); row++) {
331  auto& element = (array_buffer->at(row));
332  if (element.second) {
333  srcArrayIds[row] = *(element.second.get());
334  }
335  }
336 
339  srcArrayIds,
341 
342  for (size_t row = 0; row < array_buffer->size(); row++) {
343  auto& element = (array_buffer->at(row));
344  bool is_null = false;
345  if (element.second) {
346  *(element.second.get()) = destArrayIds[row];
347  int8_t* arrayDataPtr = reinterpret_cast<int8_t*>(&(element.second->at(0)));
348  (*arrayData)[row] = ArrayDatum(element.first * sizeof(TARGET_TYPE),
349  arrayDataPtr,
350  is_null,
351  DoNothingDeleter());
352  }
353  }
354  }
355  }
const DictDescriptor * source_dict_desc_
const DictDescriptor * target_dict_desc_
std::shared_ptr< StringDictionary > stringDict
bool is_null(const T &v, const SQLTypeInfo &t)
static void populate_string_array_ids(std::vector< std::vector< int32_t >> &dest_array_ids, StringDictionary *dest_dict, const std::vector< std::vector< int32_t >> &source_array_ids, const StringDictionary *source_dict)
NumericValueConverter< int64_t, TARGET_TYPE >::ColumnDataPtr processBuffer(ElementsBufferColumnPtr buffer)
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:120

+ Here is the call graph for this function:

template<typename TARGET_TYPE >
NumericValueConverter<int64_t, TARGET_TYPE>::ColumnDataPtr DictionaryValueConverter< TARGET_TYPE >::processBuffer ( ElementsBufferColumnPtr  buffer)
inline

Definition at line 242 of file TargetValueConvertersImpl.h.

References DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_, CHECK(), checked_malloc(), DictionaryValueConverter< TARGET_TYPE >::convertTransientStringIdToPermanentId(), DictDescriptor::dictRef, StringDictionaryProxy::getTransientMapping(), DictionaryValueConverter< TARGET_TYPE >::literals_lookup_, NumericValueConverter< int64_t, TARGET_TYPE >::null_value_, StringDictionary::populate_string_ids(), DictionaryValueConverter< TARGET_TYPE >::source_dict_desc_, DictionaryValueConverter< TARGET_TYPE >::source_dict_proxy_, DictDescriptor::stringDict, DictionaryValueConverter< TARGET_TYPE >::target_dict_desc_, and DictionaryValueConverter< TARGET_TYPE >::use_literals_.

Referenced by DictionaryValueConverter< TARGET_TYPE >::finalizeDataBlocksForInsertData(), and DictionaryValueConverter< TARGET_TYPE >::processArrayBuffer().

243  {
246  reinterpret_cast<TARGET_TYPE*>(
247  checked_malloc(buffer->size() * sizeof(TARGET_TYPE))));
248 
249  std::vector<int32_t>* bufferPtr =
250  reinterpret_cast<std::vector<int32_t>*>(buffer.get());
251  TARGET_TYPE* columnDataPtr = reinterpret_cast<TARGET_TYPE*>(data.get());
252  if (use_literals_) {
253  for (size_t i = 0; i < bufferPtr->size(); i++) {
254  auto id = literals_lookup_[(*bufferPtr)[i]];
255  if (id == buffer_null_sentinal_) {
256  columnDataPtr[i] = this->null_value_;
257  } else {
258  columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
259  }
260  }
261  } else {
263  // special case, where source and target dict are the same
264  // mostly in update
265  for (size_t i = 0; i < bufferPtr->size(); i++) {
266  auto src_id = (*bufferPtr)[i];
267  if (src_id == buffer_null_sentinal_) {
268  columnDataPtr[i] = this->null_value_;
269  } else if (src_id < 0) {
270  columnDataPtr[i] = convertTransientStringIdToPermanentId(src_id);
271  } else {
272  columnDataPtr[i] = static_cast<TARGET_TYPE>(src_id);
273  }
274  }
275 
276  } else {
277  std::vector<int32_t> dest_ids;
278  dest_ids.resize(bufferPtr->size());
279 
280  if (source_dict_proxy_) {
282  dest_ids,
284  *bufferPtr,
287  } else {
290  *bufferPtr,
292  }
293 
294  // fixup NULL sentinel
295  for (size_t i = 0; i < dest_ids.size(); i++) {
296  auto id = dest_ids[i];
297  if (id == buffer_null_sentinal_) {
298  columnDataPtr[i] = this->null_value_;
299  } else {
300  CHECK(std::numeric_limits<TARGET_TYPE>::max() >= id);
301  columnDataPtr[i] = static_cast<TARGET_TYPE>(id);
302  }
303  }
304  }
305  }
306 
307  return data;
308  }
std::unique_ptr< TARGET_TYPE, CheckedMallocDeleter< TARGET_TYPE >> ColumnDataPtr
const DictDescriptor * source_dict_desc_
static void populate_string_ids(std::vector< int32_t > &dest_ids, StringDictionary *dest_dict, const std::vector< int32_t > &source_ids, const StringDictionary *source_dict, const std::map< int32_t, std::string > transient_mapping={})
Populates provided dest_ids vector with string ids corresponding to given source strings.
const DictDescriptor * target_dict_desc_
std::shared_ptr< StringDictionary > stringDict
CHECK(cgen_state)
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
const std::map< int32_t, std::string > getTransientMapping() const
int32_t convertTransientStringIdToPermanentId(int32_t &transient_string_id)
std::unordered_map< int32_t, int32_t > literals_lookup_
StringDictionaryProxy * source_dict_proxy_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

template<typename TARGET_TYPE >
const int32_t DictionaryValueConverter< TARGET_TYPE >::buffer_null_sentinal_ = std::numeric_limits<int32_t>::min()
template<typename TARGET_TYPE >
const StringDictionaryProxy* DictionaryValueConverter< TARGET_TYPE >::literals_dict_
template<typename TARGET_TYPE >
std::unordered_map<int32_t, int32_t> DictionaryValueConverter< TARGET_TYPE >::literals_lookup_

The documentation for this struct was generated from the following file: