OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
StringDictionaryTranslationMgr Class Reference

#include <StringDictionaryTranslationMgr.h>

+ Collaboration diagram for StringDictionaryTranslationMgr:

Public Member Functions

 StringDictionaryTranslationMgr (const shared::StringDictKey &source_string_dict_key, const shared::StringDictKey &dest_string_dict_key, const bool translate_intersection_only, const SQLTypeInfo &output_ti, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos, const Data_Namespace::MemoryLevel memory_level, const int device_count, Executor *executor, Data_Namespace::DataMgr *data_mgr, const bool delay_translation)
 
 StringDictionaryTranslationMgr (const shared::StringDictKey &source_string_dict_key, const SQLTypeInfo &output_ti, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos, const Data_Namespace::MemoryLevel memory_level, const int device_count, Executor *executor, Data_Namespace::DataMgr *data_mgr, const bool delay_translation)
 
 ~StringDictionaryTranslationMgr ()
 
void buildTranslationMap ()
 
void createKernelBuffers ()
 
llvm::Value * codegen (llvm::Value *str_id_input, const SQLTypeInfo &input_ti, const bool add_nullcheck, const CompilationOptions &co) const
 
bool isMapValid () const
 
const int8_t * data () const
 
int32_t minSourceStringId () const
 
size_t mapSize () const
 

Private Attributes

const shared::StringDictKey source_string_dict_key_
 
const shared::StringDictKey dest_string_dict_key_
 
const bool translate_intersection_only_
 
const SQLTypeInfo output_ti_
 
const std::vector
< StringOps_Namespace::StringOpInfo
string_op_infos_
 
const bool has_null_string_op_
 
const Data_Namespace::MemoryLevel memory_level_
 
const int device_count_
 
Executorexecutor_
 
Data_Namespace::DataMgrdata_mgr_
 
const bool dest_type_is_string_
 
const
StringDictionaryProxy::IdMap
*host_translation_map_ 
nullptr
 
const
StringDictionaryProxy::TranslationMap
< Datum > * 
host_numeric_translation_map_
 
std::vector< const int8_t * > kernel_translation_maps_
 
std::vector
< Data_Namespace::AbstractBuffer * > 
device_buffers_
 

Detailed Description

Definition at line 48 of file StringDictionaryTranslationMgr.h.

Constructor & Destructor Documentation

StringDictionaryTranslationMgr::StringDictionaryTranslationMgr ( const shared::StringDictKey source_string_dict_key,
const shared::StringDictKey dest_string_dict_key,
const bool  translate_intersection_only,
const SQLTypeInfo output_ti,
const std::vector< StringOps_Namespace::StringOpInfo > &  string_op_infos,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_count,
Executor executor,
Data_Namespace::DataMgr data_mgr,
const bool  delay_translation 
)

Definition at line 51 of file StringDictionaryTranslationMgr.cpp.

References buildTranslationMap(), CHECK, CHECK_EQ, Data_Namespace::CPU_LEVEL, createKernelBuffers(), Data_Namespace::GPU_LEVEL, has_null_string_op_, and memory_level_.

62  : source_string_dict_key_(source_string_dict_key)
63  , dest_string_dict_key_(dest_string_dict_key)
64  , translate_intersection_only_(translate_intersection_only)
65  , output_ti_(output_ti)
66  , string_op_infos_(string_op_infos)
68  , memory_level_(memory_level)
69  , device_count_(device_count)
70  , executor_(executor)
71  , data_mgr_(data_mgr)
72  , dest_type_is_string_(true) {
73 #ifdef HAVE_CUDA
76 #else
78 #endif // HAVE_CUDA
79  if (!delay_translation && !has_null_string_op_) {
82  }
83 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
const Data_Namespace::MemoryLevel memory_level_
bool one_or_more_string_ops_is_null(const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos)
const shared::StringDictKey dest_string_dict_key_
#define CHECK(condition)
Definition: Logger.h:291
const shared::StringDictKey source_string_dict_key_
const std::vector< StringOps_Namespace::StringOpInfo > string_op_infos_

+ Here is the call graph for this function:

StringDictionaryTranslationMgr::StringDictionaryTranslationMgr ( const shared::StringDictKey source_string_dict_key,
const SQLTypeInfo output_ti,
const std::vector< StringOps_Namespace::StringOpInfo > &  string_op_infos,
const Data_Namespace::MemoryLevel  memory_level,
const int  device_count,
Executor executor,
Data_Namespace::DataMgr data_mgr,
const bool  delay_translation 
)

Definition at line 85 of file StringDictionaryTranslationMgr.cpp.

94  : source_string_dict_key_(source_string_dict_key)
95  , dest_string_dict_key_({-1, -1})
const shared::StringDictKey dest_string_dict_key_
const shared::StringDictKey source_string_dict_key_
StringDictionaryTranslationMgr::~StringDictionaryTranslationMgr ( )

Definition at line 119 of file StringDictionaryTranslationMgr.cpp.

References CHECK, data_mgr_, device_buffers_, and Data_Namespace::DataMgr::free().

119  {
120  CHECK(data_mgr_);
121  for (auto& device_buffer : device_buffers_) {
122  data_mgr_->free(device_buffer);
123  }
124 }
#define CHECK(condition)
Definition: Logger.h:291
std::vector< Data_Namespace::AbstractBuffer * > device_buffers_
void free(AbstractBuffer *buffer)
Definition: DataMgr.cpp:525

+ Here is the call graph for this function:

Member Function Documentation

void StringDictionaryTranslationMgr::buildTranslationMap ( )

Definition at line 126 of file StringDictionaryTranslationMgr.cpp.

References dest_string_dict_key_, dest_type_is_string_, executor_, host_numeric_translation_map_, RowSetMemoryOwner::SOURCE_INTERSECTION, source_string_dict_key_, RowSetMemoryOwner::SOURCE_UNION, string_op_infos_, and translate_intersection_only_.

Referenced by StringDictionaryTranslationMgr().

126  {
127  if (dest_type_is_string_) {
128  host_translation_map_ = executor_->getStringProxyTranslationMap(
135  executor_->getRowSetMemoryOwner(),
136  true);
137  } else {
139  executor_->getStringProxyNumericTranslationMap(source_string_dict_key_,
141  executor_->getRowSetMemoryOwner(),
142  true);
143  }
144 }
const StringDictionaryProxy::TranslationMap< Datum > * host_numeric_translation_map_
const shared::StringDictKey dest_string_dict_key_
const shared::StringDictKey source_string_dict_key_
const std::vector< StringOps_Namespace::StringOpInfo > string_op_infos_

+ Here is the caller graph for this function:

llvm::Value * StringDictionaryTranslationMgr::codegen ( llvm::Value *  str_id_input,
const SQLTypeInfo input_ti,
const bool  add_nullcheck,
const CompilationOptions co 
) const

Definition at line 171 of file StringDictionaryTranslationMgr.cpp.

References Parser::IntLiteral::analyzeValue(), AUTOMATIC_IR_METADATA, CHECK, CHECK_EQ, CHECK_GE, dest_type_is_string_, device_count_, CompilationOptions::device_type, executor_, SQLTypeInfo::get_notnull(), SQLTypeInfo::get_type(), GPU, Data_Namespace::GPU_LEVEL, has_null_string_op_, CompilationOptions::hoist_literals, kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kENCODING_NONE, kernel_translation_maps_, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, memory_level_, minSourceStringId(), output_ti_, to_lower(), and toString().

Referenced by CodeGenerator::codegen(), CodeGenerator::codegenCastFromString(), and CodeGenerator::codegenPseudoStringOper().

174  {
175  CHECK(kernel_translation_maps_.size() == static_cast<size_t>(device_count_) ||
177  if (!co.hoist_literals && kernel_translation_maps_.size() > 1UL) {
178  // Currently the only way to have multiple kernel translation maps is
179  // to be running on GPU, where we would need to have a different pointer
180  // per GPU to the translation map, as the address space is not shared
181  // between GPUs
182 
185 
186  // Since we currently cannot support different code per device, the only
187  // way to allow for a different kernel translation map/kernel per
188  // device(i.e. GPU) is via hoisting the map handle literal so that
189  // it can be paramertized as a kernel argument. Hence if literal
190  // hoisting is disabled (generally b/c we have an update query),
191  // the surest fire way of ensuring one and only one translation map
192  // that can have a hard-coded handle in the generated code is by running
193  // on CPU (which per the comment above currently always has a device
194  // count of 1).
195 
196  // This is not currently a major limitation as we currently run
197  // all update queries on CPU, but it would be if we want to run
198  // on multiple GPUs.
199 
200  // Todo(todd): Examine ways around the above limitation, likely either
201  // a dedicated kernel parameter for translation maps (like we have for
202  // join hash tables), or perhaps better for a number of reasons, reworking
203  // the translation map plumbing to use the join infra (which would also
204  // mean we could use pieces like the baseline hash join for multiple
205  // input string dictionaries, i.e. CONCAT on two string columns).
206 
207  throw QueryMustRunOnCpu();
208  }
209  CHECK(co.hoist_literals || kernel_translation_maps_.size() == 1UL);
210 
211  auto cgen_state_ptr = executor_->getCgenStatePtr();
212  AUTOMATIC_IR_METADATA(cgen_state_ptr);
213 
214  if (has_null_string_op_) {
215  // If any of the string ops can statically be determined to output all nulls
216  // (currently determined by whether any of the constant literal inputs to the
217  // string operation are null), then simply generate codegen a null
218  // dictionary-encoded value
219  const auto null_ti = SQLTypeInfo(kTEXT, true /* is_nullable */, kENCODING_DICT);
220  return static_cast<llvm::Value*>(executor_->cgen_state_->inlineIntNull(null_ti));
221  }
222 
223  std::vector<std::shared_ptr<const Analyzer::Constant>> constants_owned;
224  std::vector<const Analyzer::Constant*> constants;
225  for (const auto kernel_translation_map : kernel_translation_maps_) {
226  const int64_t translation_map_handle =
227  reinterpret_cast<int64_t>(kernel_translation_map);
228  const auto translation_map_handle_literal =
229  std::dynamic_pointer_cast<Analyzer::Constant>(
230  Parser::IntLiteral::analyzeValue(translation_map_handle));
231  CHECK(translation_map_handle_literal);
233  translation_map_handle_literal->get_type_info().get_compression());
234  constants_owned.push_back(translation_map_handle_literal);
235  constants.push_back(translation_map_handle_literal.get());
236  }
237  CHECK_GE(constants.size(), 1UL);
238  CHECK(co.hoist_literals || constants.size() == 1UL);
239 
240  CodeGenerator code_generator(executor_);
241 
242  const auto translation_map_handle_lvs =
243  co.hoist_literals
244  ? code_generator.codegenHoistedConstants(constants, kENCODING_NONE, {})
245  : code_generator.codegen(constants[0], false, co);
246  CHECK_EQ(size_t(1), translation_map_handle_lvs.size());
247 
248  std::unique_ptr<CodeGenerator::NullCheckCodegen> nullcheck_codegen;
249  const bool is_nullable = !input_ti.get_notnull();
250  const auto decoded_input_ti = SQLTypeInfo(kTEXT, is_nullable, kENCODING_DICT);
251  if (add_nullcheck && is_nullable) {
252  nullcheck_codegen = std::make_unique<CodeGenerator::NullCheckCodegen>(
253  cgen_state_ptr,
254  executor_,
255  input_str_id_lv,
256  decoded_input_ti,
257  "dict_encoded_str_cast_nullcheck");
258  }
259  llvm::Value* ret;
260  if (dest_type_is_string_) {
261  ret = cgen_state_ptr->emitCall(
262  "map_string_dict_id",
263  {input_str_id_lv,
264  cgen_state_ptr->castToTypeIn(translation_map_handle_lvs.front(), 64),
265  cgen_state_ptr->llInt(minSourceStringId())});
266  } else {
267  std::string fn_call = "map_string_to_datum_";
268  const auto sql_type = output_ti_.get_type();
269  switch (sql_type) {
270  case kBOOLEAN: {
271  fn_call += "bool";
272  break;
273  }
274  case kTINYINT:
275  case kSMALLINT:
276  case kINT:
277  case kBIGINT:
278  case kFLOAT:
279  case kDOUBLE: {
280  fn_call += to_lower(toString(sql_type));
281  break;
282  }
283  case kNUMERIC:
284  case kDECIMAL:
285  case kTIME:
286  case kTIMESTAMP:
287  case kDATE: {
288  fn_call += "bigint";
289  break;
290  }
291  default: {
292  throw std::runtime_error("Unimplemented type for string-to-numeric translation");
293  }
294  }
295  ret = cgen_state_ptr->emitCall(
296  fn_call,
297  {input_str_id_lv,
298  cgen_state_ptr->castToTypeIn(translation_map_handle_lvs.front(), 64),
299  cgen_state_ptr->llInt(minSourceStringId())});
300  }
301 
302  if (nullcheck_codegen) {
303  ret = nullcheck_codegen->finalize(cgen_state_ptr->inlineNull(output_ti_), ret);
304  }
305  return ret;
306 }
std::string to_lower(const std::string &str)
#define CHECK_EQ(x, y)
Definition: Logger.h:301
const Data_Namespace::MemoryLevel memory_level_
Definition: sqltypes.h:66
static std::shared_ptr< Analyzer::Expr > analyzeValue(const int64_t intval)
Definition: ParserNode.cpp:162
#define CHECK_GE(x, y)
Definition: Logger.h:306
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:381
std::string toString(const ExecutorDeviceType &device_type)
#define AUTOMATIC_IR_METADATA(CGENSTATE)
ExecutorDeviceType device_type
Definition: sqltypes.h:69
Definition: sqltypes.h:70
std::vector< const int8_t * > kernel_translation_maps_
#define CHECK(condition)
Definition: Logger.h:291
Definition: sqltypes.h:62
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:388

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void StringDictionaryTranslationMgr::createKernelBuffers ( )

Definition at line 146 of file StringDictionaryTranslationMgr.cpp.

References CudaAllocator::allocGpuAbstractBuffer(), CHECK_EQ, copy_to_nvidia_gpu(), Data_Namespace::CPU_LEVEL, data(), data_mgr_, device_buffers_, device_count_, Data_Namespace::GPU_LEVEL, kernel_translation_maps_, mapSize(), and memory_level_.

Referenced by StringDictionaryTranslationMgr().

146  {
147 #ifdef HAVE_CUDA
149  const size_t translation_map_size_bytes = mapSize();
150  for (int device_id = 0; device_id < device_count_; ++device_id) {
152  data_mgr_, translation_map_size_bytes, device_id));
153  auto device_buffer =
154  reinterpret_cast<int8_t*>(device_buffers_.back()->getMemoryPtr());
156  reinterpret_cast<CUdeviceptr>(device_buffer),
157  data(),
158  translation_map_size_bytes,
159  device_id);
160  kernel_translation_maps_.push_back(device_buffer);
161  }
162  }
163 #else
164  CHECK_EQ(1, device_count_);
165 #endif // HAVE_CUDA
167  kernel_translation_maps_.push_back(data());
168  }
169 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
const Data_Namespace::MemoryLevel memory_level_
std::vector< const int8_t * > kernel_translation_maps_
static Data_Namespace::AbstractBuffer * allocGpuAbstractBuffer(Data_Namespace::DataMgr *data_mgr, const size_t num_bytes, const int device_id)
void copy_to_nvidia_gpu(Data_Namespace::DataMgr *data_mgr, CUdeviceptr dst, const void *src, const size_t num_bytes, const int device_id)
Definition: GpuMemUtils.cpp:35
std::vector< Data_Namespace::AbstractBuffer * > device_buffers_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int8_t * StringDictionaryTranslationMgr::data ( ) const

Definition at line 316 of file StringDictionaryTranslationMgr.cpp.

References StringDictionaryProxy::TranslationMap< T >::data(), dest_type_is_string_, host_numeric_translation_map_, and isMapValid().

Referenced by createKernelBuffers().

316  {
317  if (isMapValid()) {
318  if (dest_type_is_string_) {
319  return reinterpret_cast<const int8_t*>(host_translation_map_->data());
320  } else {
321  return reinterpret_cast<const int8_t*>(host_numeric_translation_map_->data());
322  }
323  }
324  return nullptr;
325 }
const StringDictionaryProxy::TranslationMap< Datum > * host_numeric_translation_map_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool StringDictionaryTranslationMgr::isMapValid ( ) const

Definition at line 308 of file StringDictionaryTranslationMgr.cpp.

References dest_type_is_string_, StringDictionaryProxy::TranslationMap< T >::empty(), and host_numeric_translation_map_.

Referenced by data(), mapSize(), and minSourceStringId().

308  {
309  if (dest_type_is_string_) {
310  return host_translation_map_ && !host_translation_map_->empty();
311  } else {
313  }
314 }
const StringDictionaryProxy::TranslationMap< Datum > * host_numeric_translation_map_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t StringDictionaryTranslationMgr::mapSize ( ) const

Definition at line 335 of file StringDictionaryTranslationMgr.cpp.

References dest_type_is_string_, SQLTypeInfo::get_logical_size(), StringDictionaryProxy::TranslationMap< T >::getVectorMap(), host_numeric_translation_map_, isMapValid(), and output_ti_.

Referenced by createKernelBuffers().

335  {
336  if (isMapValid()) {
337  const size_t num_elems = dest_type_is_string_
338  ? host_translation_map_->getVectorMap().size()
340  const size_t elem_size =
342  return num_elems * elem_size;
343  }
344  return 0UL;
345 }
const StringDictionaryProxy::TranslationMap< Datum > * host_numeric_translation_map_
int get_logical_size() const
Definition: sqltypes.h:403
std::vector< T > const & getVectorMap() const
Definition: Datum.h:67

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t StringDictionaryTranslationMgr::minSourceStringId ( ) const

Definition at line 327 of file StringDictionaryTranslationMgr.cpp.

References dest_type_is_string_, StringDictionaryProxy::TranslationMap< T >::domainStart(), host_numeric_translation_map_, and isMapValid().

Referenced by codegen().

327  {
328  if (isMapValid()) {
329  return dest_type_is_string_ ? host_translation_map_->domainStart()
331  }
332  return 0;
333 }
const StringDictionaryProxy::TranslationMap< Datum > * host_numeric_translation_map_

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Member Data Documentation

Data_Namespace::DataMgr* StringDictionaryTranslationMgr::data_mgr_
private
const shared::StringDictKey StringDictionaryTranslationMgr::dest_string_dict_key_
private

Definition at line 87 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().

const bool StringDictionaryTranslationMgr::dest_type_is_string_
private
std::vector<Data_Namespace::AbstractBuffer*> StringDictionaryTranslationMgr::device_buffers_
private
const int StringDictionaryTranslationMgr::device_count_
private

Definition at line 93 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and createKernelBuffers().

Executor* StringDictionaryTranslationMgr::executor_
private

Definition at line 94 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap(), and codegen().

const bool StringDictionaryTranslationMgr::has_null_string_op_
private

Definition at line 91 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and StringDictionaryTranslationMgr().

const StringDictionaryProxy::TranslationMap<Datum>* StringDictionaryTranslationMgr::host_numeric_translation_map_
private
Initial value:
{
nullptr}

Definition at line 98 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap(), data(), isMapValid(), mapSize(), and minSourceStringId().

std::vector<const int8_t*> StringDictionaryTranslationMgr::kernel_translation_maps_
private

Definition at line 100 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and createKernelBuffers().

const Data_Namespace::MemoryLevel StringDictionaryTranslationMgr::memory_level_
private
const StringDictionaryProxy::IdMap* host_translation_map_ StringDictionaryTranslationMgr::nullptr
private

Definition at line 97 of file StringDictionaryTranslationMgr.h.

const SQLTypeInfo StringDictionaryTranslationMgr::output_ti_
private

Definition at line 89 of file StringDictionaryTranslationMgr.h.

Referenced by codegen(), and mapSize().

const shared::StringDictKey StringDictionaryTranslationMgr::source_string_dict_key_
private

Definition at line 86 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().

const std::vector<StringOps_Namespace::StringOpInfo> StringDictionaryTranslationMgr::string_op_infos_
private

Definition at line 90 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().

const bool StringDictionaryTranslationMgr::translate_intersection_only_
private

Definition at line 88 of file StringDictionaryTranslationMgr.h.

Referenced by buildTranslationMap().


The documentation for this class was generated from the following files: