OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
TableFunctionCompilationContext.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #include <llvm/IR/InstIterator.h>
20 #include <llvm/IR/Verifier.h>
21 #include <llvm/Support/raw_os_ostream.h>
22 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
23 #include <algorithm>
24 #include <boost/algorithm/string.hpp>
25 
27 
28 namespace {
29 
30 llvm::Function* generate_entry_point(const CgenState* cgen_state) {
31  auto& ctx = cgen_state->context_;
32  const auto pi8_type = llvm::PointerType::get(get_int_type(8, ctx), 0);
33  const auto ppi8_type = llvm::PointerType::get(pi8_type, 0);
34  const auto pi64_type = llvm::PointerType::get(get_int_type(64, ctx), 0);
35  const auto ppi64_type = llvm::PointerType::get(pi64_type, 0);
36  const auto i32_type = get_int_type(32, ctx);
37 
38  const auto func_type = llvm::FunctionType::get(
39  i32_type,
40  {pi8_type, ppi8_type, pi64_type, ppi8_type, ppi64_type, ppi8_type, pi64_type},
41  false);
42 
43  auto func = llvm::Function::Create(func_type,
44  llvm::Function::ExternalLinkage,
45  "call_table_function",
46  cgen_state->module_);
47  auto arg_it = func->arg_begin();
48  const auto mgr_arg = &*arg_it;
49  mgr_arg->setName("mgr_ptr");
50  const auto input_cols_arg = &*(++arg_it);
51  input_cols_arg->setName("input_col_buffers");
52  const auto input_row_counts = &*(++arg_it);
53  input_row_counts->setName("input_row_counts");
54  const auto input_str_dict_proxies = &*(++arg_it);
55  input_str_dict_proxies->setName("input_str_dict_proxies");
56  const auto output_buffers = &*(++arg_it);
57  output_buffers->setName("output_buffers");
58  const auto output_str_dict_proxies = &*(++arg_it);
59  output_str_dict_proxies->setName("output_str_dict_proxies");
60  const auto output_row_count = &*(++arg_it);
61  output_row_count->setName("output_row_count");
62  return func;
63 }
64 
66  llvm::LLVMContext& ctx) {
67  if (elem_ti.is_fp()) {
68  return get_fp_ptr_type(elem_ti.get_size() * 8, ctx);
69  } else if (elem_ti.is_boolean()) {
70  return get_int_ptr_type(8, ctx);
71  } else if (elem_ti.is_integer()) {
72  return get_int_ptr_type(elem_ti.get_size() * 8, ctx);
73  } else if (elem_ti.is_string()) {
74  if (elem_ti.get_compression() == kENCODING_DICT) {
75  return get_int_ptr_type(elem_ti.get_size() * 8, ctx);
76  }
77  CHECK(elem_ti.is_bytes()); // None encoded string
78  return get_int_ptr_type(8, ctx);
79  } else if (elem_ti.is_timestamp()) {
80  return get_int_ptr_type(elem_ti.get_size() * 8, ctx);
81  }
82  LOG(FATAL) << "get_llvm_type_from_sql_column_type: not implemented for "
83  << ::toString(elem_ti);
84  return nullptr;
85 }
86 
87 std::tuple<llvm::Value*, llvm::Value*> alloc_column(std::string col_name,
88  const size_t index,
89  const SQLTypeInfo& data_target_info,
90  llvm::Value* data_ptr,
91  llvm::Value* data_size,
92  llvm::Value* data_str_dict_proxy_ptr,
93  llvm::LLVMContext& ctx,
94  llvm::IRBuilder<>& ir_builder) {
95  /*
96  Creates a new Column instance of given element type and initialize
97  its data ptr and sz members when specified. If data ptr or sz are
98  unspecified (have nullptr values) then the corresponding members
99  are initialized with NULL and -1, respectively.
100 
101  If we are allocating a TextEncodingDict Column type, this function
102  adds and populates a int8* pointer to a StringDictProxy object.
103 
104  Return a pair of Column allocation (caller should apply
105  builder.CreateLoad to it in order to construct a Column instance
106  as a value) and a pointer to the Column instance.
107  */
108  llvm::Type* data_ptr_llvm_type =
109  get_llvm_type_from_sql_column_type(data_target_info, ctx);
110  const bool is_text_encoding_dict_type =
111  data_target_info.is_string() &&
112  data_target_info.get_compression() == kENCODING_DICT;
113 
114  llvm::StructType* col_struct_type =
115  is_text_encoding_dict_type
116  ? llvm::StructType::get(
117  ctx,
118  {
119  data_ptr_llvm_type, /* T* ptr */
120  llvm::Type::getInt64Ty(ctx), /* int64_t sz */
121  llvm::Type::getInt8PtrTy(ctx) /* int64_t string_dictionary_ptr */
122  })
123  : llvm::StructType::get(ctx,
124  {
125  data_ptr_llvm_type, /* T* ptr */
126  llvm::Type::getInt64Ty(ctx) /* int64_t sz */
127  });
128  auto col = ir_builder.CreateAlloca(col_struct_type);
129  col->setName(col_name);
130  auto col_ptr_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 0);
131  auto col_sz_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 1);
132  auto col_str_dict_ptr = is_text_encoding_dict_type
133  ? ir_builder.CreateStructGEP(col_struct_type, col, 2)
134  : nullptr;
135  col_ptr_ptr->setName(col_name + ".ptr");
136  col_sz_ptr->setName(col_name + ".sz");
137  if (is_text_encoding_dict_type) {
138  col_str_dict_ptr->setName(col_name + ".string_dict_proxy");
139  }
140 
141  if (data_ptr != nullptr) {
142  if (data_ptr->getType() == data_ptr_llvm_type->getPointerElementType()) {
143  ir_builder.CreateStore(data_ptr, col_ptr_ptr);
144  } else {
145  auto tmp = ir_builder.CreateBitCast(data_ptr, data_ptr_llvm_type);
146  ir_builder.CreateStore(tmp, col_ptr_ptr);
147  }
148  } else {
149  ir_builder.CreateStore(llvm::Constant::getNullValue(data_ptr_llvm_type), col_ptr_ptr);
150  }
151  llvm::Value* size_val = nullptr;
152  if (data_size != nullptr) {
153  auto data_size_type = data_size->getType();
154  if (data_size_type->isPointerTy()) {
155  CHECK(data_size_type->getPointerElementType()->isIntegerTy(64));
156  size_val =
157  ir_builder.CreateLoad(data_size->getType()->getPointerElementType(), data_size);
158  } else {
159  CHECK(data_size_type->isIntegerTy(64));
160  size_val = data_size;
161  }
162  ir_builder.CreateStore(size_val, col_sz_ptr);
163  } else {
164  auto const_minus1 = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), -1, true);
165  ir_builder.CreateStore(const_minus1, col_sz_ptr);
166  }
167  if (is_text_encoding_dict_type) {
168  if (data_str_dict_proxy_ptr != nullptr) {
169  auto data_str_dict_proxy_type = data_str_dict_proxy_ptr->getType();
170  CHECK(data_str_dict_proxy_type->getPointerElementType()->isIntegerTy(8));
171  ir_builder.CreateStore(data_str_dict_proxy_ptr, col_str_dict_ptr);
172  } else {
173  ir_builder.CreateStore(llvm::Constant::getNullValue(llvm::Type::getInt8PtrTy(ctx)),
174  col_str_dict_ptr);
175  }
176  }
177 
178  auto col_ptr = ir_builder.CreatePointerCast(
179  col_ptr_ptr, llvm::PointerType::get(llvm::Type::getInt8Ty(ctx), 0));
180  col_ptr->setName(col_name + "_ptr");
181  return {col, col_ptr};
182 }
183 
184 llvm::Value* alloc_column_list(std::string col_list_name,
185  const SQLTypeInfo& data_target_info,
186  llvm::Value* data_ptrs,
187  int length,
188  llvm::Value* data_size,
189  llvm::Value* data_str_dict_proxy_ptrs,
190  llvm::LLVMContext& ctx,
191  llvm::IRBuilder<>& ir_builder) {
192  /*
193  Creates a new ColumnList instance of given element type and initialize
194  its members. If data ptr or size are unspecified (have nullptr
195  values) then the corresponding members are initialized with NULL
196  and -1, respectively.
197  */
198  llvm::Type* data_ptrs_llvm_type = llvm::Type::getInt8PtrTy(ctx);
199  const bool is_text_encoding_dict_type =
200  data_target_info.is_string() &&
201  data_target_info.get_compression() == kENCODING_DICT;
202 
203  llvm::StructType* col_list_struct_type =
204  is_text_encoding_dict_type
205  ? llvm::StructType::get(
206  ctx,
207  {
208  data_ptrs_llvm_type, /* int8_t* ptrs */
209  llvm::Type::getInt64Ty(ctx), /* int64_t length */
210  llvm::Type::getInt64Ty(ctx), /* int64_t size */
211  data_ptrs_llvm_type /* int8_t* str_dict_proxy_ptrs */
212  })
213  : llvm::StructType::get(ctx,
214  {
215  data_ptrs_llvm_type, /* int8_t* ptrs */
216  llvm::Type::getInt64Ty(ctx), /* int64_t length */
217  llvm::Type::getInt64Ty(ctx) /* int64_t size */
218  });
219 
220  auto col_list = ir_builder.CreateAlloca(col_list_struct_type);
221  col_list->setName(col_list_name);
222  auto col_list_ptr_ptr = ir_builder.CreateStructGEP(col_list_struct_type, col_list, 0);
223  auto col_list_length_ptr =
224  ir_builder.CreateStructGEP(col_list_struct_type, col_list, 1);
225  auto col_list_size_ptr = ir_builder.CreateStructGEP(col_list_struct_type, col_list, 2);
226  auto col_str_dict_ptr_ptr =
227  is_text_encoding_dict_type
228  ? ir_builder.CreateStructGEP(col_list_struct_type, col_list, 3)
229  : nullptr;
230 
231  col_list_ptr_ptr->setName(col_list_name + ".ptrs");
232  col_list_length_ptr->setName(col_list_name + ".length");
233  col_list_size_ptr->setName(col_list_name + ".size");
234  if (is_text_encoding_dict_type) {
235  col_str_dict_ptr_ptr->setName(col_list_name + ".string_dict_proxies");
236  }
237 
238  CHECK(length >= 0);
239  auto const_length = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), length, true);
240 
241  if (data_ptrs != nullptr) {
242  if (data_ptrs->getType() == data_ptrs_llvm_type->getPointerElementType()) {
243  ir_builder.CreateStore(data_ptrs, col_list_ptr_ptr);
244  } else {
245  auto tmp = ir_builder.CreateBitCast(data_ptrs, data_ptrs_llvm_type);
246  ir_builder.CreateStore(tmp, col_list_ptr_ptr);
247  }
248  } else {
249  ir_builder.CreateStore(llvm::Constant::getNullValue(data_ptrs_llvm_type),
250  col_list_ptr_ptr);
251  }
252 
253  ir_builder.CreateStore(const_length, col_list_length_ptr);
254 
255  if (data_size != nullptr) {
256  auto data_size_type = data_size->getType();
257  llvm::Value* size_val = nullptr;
258  if (data_size_type->isPointerTy()) {
259  CHECK(data_size_type->getPointerElementType()->isIntegerTy(64));
260  size_val =
261  ir_builder.CreateLoad(data_size->getType()->getPointerElementType(), data_size);
262  } else {
263  CHECK(data_size_type->isIntegerTy(64));
264  size_val = data_size;
265  }
266  ir_builder.CreateStore(size_val, col_list_size_ptr);
267  } else {
268  auto const_minus1 = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), -1, true);
269  ir_builder.CreateStore(const_minus1, col_list_size_ptr);
270  }
271 
272  if (is_text_encoding_dict_type) {
273  if (data_str_dict_proxy_ptrs != nullptr) {
274  auto data_str_dict_proxies_type = data_str_dict_proxy_ptrs->getType();
275  CHECK(data_str_dict_proxies_type->getPointerElementType()->isIntegerTy(8));
276  ir_builder.CreateStore(data_str_dict_proxy_ptrs, col_str_dict_ptr_ptr);
277  } else {
278  ir_builder.CreateStore(llvm::Constant::getNullValue(data_ptrs_llvm_type),
279  col_list_ptr_ptr);
280  }
281  }
282 
283  auto col_list_ptr = ir_builder.CreatePointerCast(
284  col_list_ptr_ptr, llvm::PointerType::get(llvm::Type::getInt8Ty(ctx), 0));
285  col_list_ptr->setName(col_list_name + "_ptrs");
286  return col_list_ptr;
287 }
288 
289 static bool columnTypeRequiresCasting(const SQLTypeInfo& ti) {
290  /*
291  Returns whether a column requires casting before table function execution based on its
292  underlying SQL type
293  */
294 
295  if (!ti.is_column()) {
296  return false;
297  }
298 
299  // TIMESTAMP columns should always have nanosecond precision
300  if (ti.get_subtype() == kTIMESTAMP && ti.get_precision() != 9) {
301  return true;
302  }
303 
304  return false;
305 }
306 
307 llvm::Value* cast_value(llvm::Value* value,
308  SQLTypeInfo& orig_ti,
309  SQLTypeInfo& dest_ti,
310  bool nullable,
311  CodeGenerator& codeGenerator) {
312  /*
313  Codegens a cast of a value from a given origin type to a given destination type, if such
314  implementation is available. Errors for unsupported casts.
315  */
316  if (orig_ti.is_timestamp() && dest_ti.is_timestamp()) {
317  return codeGenerator.codegenCastBetweenTimestamps(
318  value, orig_ti, dest_ti, !dest_ti.get_notnull());
319  } else {
320  throw std::runtime_error("Unsupported cast from " + orig_ti.get_type_name() + " to " +
321  dest_ti.get_type_name() + " during UDTF code generation.");
322  }
323 }
324 
325 void cast_column(llvm::Value* col_base_ptr,
326  llvm::Function* func,
327  SQLTypeInfo& orig_ti,
328  SQLTypeInfo& dest_ti,
329  std::string index,
330  llvm::IRBuilder<>& ir_builder,
331  llvm::LLVMContext& ctx,
332  CodeGenerator& codeGenerator) {
333  /*
334  Generates code to cast a Column instance from a given origin
335  SQLType to a new destinaton SQLType. To do so, it generates a
336  loop with the following overall structure:
337 
338  --------------
339  | pre_header |
340  | i = 0 |
341  --------------
342  |
343  v
344  ---------------- ----------------
345  | cond | | body |
346  | i < col.size | (True) -> | cast(col[i]) |
347  ---------------- | i++ |
348  (False) ^ ----------------
349  | \____________________/
350  |
351  v
352  ---------------
353  | end |
354  ---------------
355 
356  The correctness of the cast as well as error handling/early
357  exiting in case of cast failures are left to the CodeGenerator
358  which generates the code for the cast operation itself.
359  */
360 
361  llvm::BasicBlock* for_pre =
362  llvm::BasicBlock::Create(ctx, "for_pre_cast." + index, func);
363  llvm::BasicBlock* for_cond =
364  llvm::BasicBlock::Create(ctx, "for_cond_cast." + index, func);
365  llvm::BasicBlock* for_body =
366  llvm::BasicBlock::Create(ctx, "for_body_cast." + index, func);
367  llvm::BasicBlock* for_end =
368  llvm::BasicBlock::Create(ctx, "for_end_cast." + index, func);
369  ir_builder.CreateBr(for_pre);
370 
371  // pre-header: load column ptr and size
372  ir_builder.SetInsertPoint(for_pre);
373  llvm::Type* data_type = get_llvm_type_from_sql_column_type(orig_ti, ctx);
374  llvm::StructType* col_struct_type =
375  llvm::StructType::get(ctx, {data_type, ir_builder.getInt64Ty()});
376  llvm::Value* struct_cast = ir_builder.CreateBitCast(
377  col_base_ptr, col_struct_type->getPointerTo(), "col_struct." + index);
378  llvm::Value* gep_ptr = ir_builder.CreateStructGEP(
379  col_struct_type, struct_cast, 0, "col_ptr_addr." + index);
380  llvm::Value* col_ptr = ir_builder.CreateLoad(data_type, gep_ptr, "col_ptr." + index);
381  llvm::Value* gep_sz =
382  ir_builder.CreateStructGEP(col_struct_type, struct_cast, 1, "col_sz_addr." + index);
383  llvm::Value* col_sz =
384  ir_builder.CreateLoad(ir_builder.getInt64Ty(), gep_sz, "col_sz." + index);
385  ir_builder.CreateBr(for_cond);
386 
387  // condition: check induction variable against loop predicate
388  ir_builder.SetInsertPoint(for_cond);
389  llvm::PHINode* for_ind_var =
390  ir_builder.CreatePHI(ir_builder.getInt64Ty(), 2, "for_ind_var." + index);
391  for_ind_var->addIncoming(ir_builder.getInt64(0), for_pre);
392  llvm::Value* for_pred =
393  ir_builder.CreateICmpSLT(for_ind_var, col_sz, "for_pred." + index);
394  ir_builder.CreateCondBr(for_pred, for_body, for_end);
395 
396  // body: perform value cast, increment induction variable
397  ir_builder.SetInsertPoint(for_body);
398  ;
399  llvm::Value* val_gep = ir_builder.CreateInBoundsGEP(
400  ir_builder.getInt64Ty(), col_ptr, for_ind_var, "val_gep." + index);
401  llvm::Value* val_load =
402  ir_builder.CreateLoad(ir_builder.getInt64Ty(), val_gep, "val_load." + index);
403  llvm::Value* cast_result = cast_value(val_load, orig_ti, dest_ti, false, codeGenerator);
404  cast_result->setName("cast_result." + index);
405  ir_builder.CreateStore(cast_result, val_gep);
406  llvm::Value* for_inc =
407  ir_builder.CreateAdd(for_ind_var, ir_builder.getInt64(1), "for_inc." + index);
408  ir_builder.CreateBr(for_cond);
409  // the cast codegening may have generated extra blocks, so for_body does not necessarily
410  // jump to for_cond directly
411  llvm::Instruction* inc_as_inst = llvm::cast<llvm::Instruction>(for_inc);
412  for_ind_var->addIncoming(for_inc, inc_as_inst->getParent());
413  ir_builder.SetInsertPoint(for_end);
414 }
415 
416 std::string exprsKey(const std::vector<Analyzer::Expr*>& exprs) {
417  std::string result;
418  for (const auto& expr : exprs) {
419  const auto& ti = expr->get_type_info();
420  result += ti.to_string() + ", ";
421  }
422  return result;
423 }
424 
425 } // namespace
426 
427 std::shared_ptr<CompilationContext> TableFunctionCompilationContext::compile(
428  const TableFunctionExecutionUnit& exe_unit,
429  bool emit_only_preflight_fn) {
430  auto timer = DEBUG_TIMER(__func__);
431 
432  // Here we assume that Executor::tf_code_accessor is cleared when a
433  // UDTF implementation is changed. TODO: Ideally, the key should
434  // contain a hash of an UDTF implementation string. This could be
435  // achieved by including the hash value to the prefix of the UDTF
436  // name, for instance.
437  CodeCacheKey key{exe_unit.table_func.getName(),
438  exprsKey(exe_unit.input_exprs),
439  exprsKey(exe_unit.target_exprs),
440  std::to_string(emit_only_preflight_fn),
442 
443  auto cached_code = Executor::tf_code_accessor.get_or_wait(key);
444  if (cached_code) {
445  return *cached_code;
446  }
447 
448  auto cgen_state = executor_->getCgenStatePtr();
449  CHECK(cgen_state);
450  CHECK(cgen_state->module_ == nullptr);
451  cgen_state->set_module_shallow_copy(executor_->get_rt_module());
452 
454 
455  generateEntryPoint(exe_unit, emit_only_preflight_fn);
456 
458  CHECK(!emit_only_preflight_fn);
460  }
461 
462  Executor::tf_code_accessor.swap(key, finalize(emit_only_preflight_fn));
463  return Executor::tf_code_accessor.get_value(key);
464 }
465 
467  const TableFunctionExecutionUnit& exe_unit) {
468  bool is_gpu = co_.device_type == ExecutorDeviceType::GPU;
469  auto mod = executor_->get_rt_udf_module(is_gpu).get();
470  if (mod != nullptr) {
471  auto* flag = mod->getModuleFlag("pass_column_arguments_by_value");
472  if (auto* cnt = llvm::mdconst::extract_or_null<llvm::ConstantInt>(flag)) {
473  return cnt->getZExtValue();
474  }
475  }
476 
477  // fallback to original behavior
478  return exe_unit.table_func.isRuntime();
479 }
480 
482  const TableFunctionExecutionUnit& exe_unit,
483  const std::vector<llvm::Value*>& func_args,
484  llvm::BasicBlock* bb_exit,
485  llvm::Value* output_row_count_ptr,
486  bool emit_only_preflight_fn) {
487  auto cgen_state = executor_->getCgenStatePtr();
488  // Emit llvm IR code to call the table function
489  llvm::LLVMContext& ctx = cgen_state->context_;
490  llvm::IRBuilder<>* ir_builder = &cgen_state->ir_builder_;
491 
492  std::string func_name =
493  (emit_only_preflight_fn ? exe_unit.table_func.getPreFlightFnName()
494  : exe_unit.table_func.getName(false, true));
495  llvm::Value* table_func_return =
496  cgen_state->emitExternalCall(func_name, get_int_type(32, ctx), func_args);
497 
498  table_func_return->setName(emit_only_preflight_fn ? "preflight_check_func_ret"
499  : "table_func_ret");
500 
501  // If table_func_return is non-negative then store the value in
502  // output_row_count and return zero. Otherwise, return
503  // table_func_return that negative value contains the error code.
504  llvm::BasicBlock* bb_exit_0 =
505  llvm::BasicBlock::Create(ctx, ".exit0", entry_point_func_);
506 
507  llvm::Constant* const_zero =
508  llvm::ConstantInt::get(table_func_return->getType(), 0, true);
509  llvm::Value* is_ok = ir_builder->CreateICmpSGE(table_func_return, const_zero);
510  ir_builder->CreateCondBr(is_ok, bb_exit_0, bb_exit);
511 
512  ir_builder->SetInsertPoint(bb_exit_0);
513  llvm::Value* r =
514  ir_builder->CreateIntCast(table_func_return, get_int_type(64, ctx), true);
515  ir_builder->CreateStore(r, output_row_count_ptr);
516  ir_builder->CreateRet(const_zero);
517 
518  ir_builder->SetInsertPoint(bb_exit);
519  ir_builder->CreateRet(table_func_return);
520 }
521 
523  const TableFunctionExecutionUnit& exe_unit,
524  bool emit_only_preflight_fn) {
525  auto timer = DEBUG_TIMER(__func__);
527  CHECK_EQ(entry_point_func_->arg_size(), 7);
528  auto arg_it = entry_point_func_->arg_begin();
529  const auto mgr_ptr = &*arg_it;
530  const auto input_cols_arg = &*(++arg_it);
531  const auto input_row_counts_arg = &*(++arg_it);
532  const auto input_str_dict_proxies_arg = &*(++arg_it);
533  const auto output_buffers_arg = &*(++arg_it);
534  const auto output_str_dict_proxies_arg = &*(++arg_it);
535  const auto output_row_count_ptr = &*(++arg_it);
536  auto cgen_state = executor_->getCgenStatePtr();
537  CHECK(cgen_state);
538  auto& ctx = cgen_state->context_;
539 
540  llvm::BasicBlock* bb_entry =
541  llvm::BasicBlock::Create(ctx, ".entry", entry_point_func_, 0);
542  cgen_state->ir_builder_.SetInsertPoint(bb_entry);
543 
544  llvm::BasicBlock* bb_exit = llvm::BasicBlock::Create(ctx, ".exit", entry_point_func_);
545 
546  llvm::BasicBlock* func_body_bb = llvm::BasicBlock::Create(
547  ctx, ".func_body0", cgen_state->ir_builder_.GetInsertBlock()->getParent());
548 
549  cgen_state->ir_builder_.SetInsertPoint(bb_entry);
550  cgen_state->ir_builder_.CreateBr(func_body_bb);
551 
552  cgen_state->ir_builder_.SetInsertPoint(func_body_bb);
553  auto col_heads = generate_column_heads_load(
554  exe_unit.input_exprs.size(), input_cols_arg, cgen_state->ir_builder_, ctx);
555  CHECK_EQ(exe_unit.input_exprs.size(), col_heads.size());
556  auto row_count_heads = generate_column_heads_load(
557  exe_unit.input_exprs.size(), input_row_counts_arg, cgen_state->ir_builder_, ctx);
558 
559  auto input_str_dict_proxy_heads = std::vector<llvm::Value*>();
560  if (!emit_only_preflight_fn and co_.device_type == ExecutorDeviceType::CPU) {
561  input_str_dict_proxy_heads = generate_column_heads_load(exe_unit.input_exprs.size(),
562  input_str_dict_proxies_arg,
563  cgen_state->ir_builder_,
564  ctx);
565  }
566  // The column arguments of C++ UDTFs processed by clang must be
567  // passed by reference, see rbc issues 200 and 289.
568  auto pass_column_by_value = passColumnsByValue(exe_unit);
569  std::vector<llvm::Value*> func_args;
570  std::vector<std::pair<llvm::Value*, const SQLTypeInfo>> columns_to_cast;
571  size_t func_arg_index = 0;
572  if (exe_unit.table_func.usesManager()) {
573  func_args.push_back(mgr_ptr);
574  func_arg_index++;
575  }
576  int col_index = -1;
577 
578  for (size_t i = 0; i < exe_unit.input_exprs.size(); i++) {
579  const auto& expr = exe_unit.input_exprs[i];
580  const auto& ti = expr->get_type_info();
581  if (col_index == -1) {
582  func_arg_index += 1;
583  }
584  if (ti.is_fp()) {
585  auto r = cgen_state->ir_builder_.CreateBitCast(
586  col_heads[i], get_fp_ptr_type(get_bit_width(ti), ctx));
587  llvm::LoadInst* scalar_fp = cgen_state->ir_builder_.CreateLoad(
588  r->getType()->getPointerElementType(),
589  r,
590  "input_scalar_fp." + std::to_string(func_arg_index));
591  func_args.push_back(scalar_fp);
592  CHECK_EQ(col_index, -1);
593  } else if (ti.is_integer() || ti.is_boolean() || ti.is_timestamp()) {
594  auto r = cgen_state->ir_builder_.CreateBitCast(
595  col_heads[i], get_int_ptr_type(get_bit_width(ti), ctx));
596  llvm::LoadInst* scalar_int = cgen_state->ir_builder_.CreateLoad(
597  r->getType()->getPointerElementType(),
598  r,
599  "input_scalar_int." + std::to_string(func_arg_index));
600  func_args.push_back(scalar_int);
601  CHECK_EQ(col_index, -1);
602  } else if (ti.is_bytes()) {
603  auto varchar_size =
604  cgen_state->ir_builder_.CreateBitCast(col_heads[i], get_int_ptr_type(64, ctx));
605  auto varchar_ptr = cgen_state->ir_builder_.CreateGEP(
606  col_heads[i]->getType()->getScalarType()->getPointerElementType(),
607  col_heads[i],
608  cgen_state->llInt(8));
609  auto [varchar_struct, varchar_struct_ptr] = alloc_column(
610  std::string("input_varchar_literal.") + std::to_string(func_arg_index),
611  i,
612  ti,
613  varchar_ptr,
614  varchar_size,
615  nullptr,
616  ctx,
617  cgen_state->ir_builder_);
618  func_args.push_back(
619  (pass_column_by_value
620  ? cgen_state->ir_builder_.CreateLoad(
621  varchar_struct->getType()->getPointerElementType(), varchar_struct)
622  : varchar_struct_ptr));
623  CHECK_EQ(col_index, -1);
624  } else if (ti.is_column()) {
625  auto [col, col_ptr] = alloc_column(
626  std::string("input_col.") + std::to_string(func_arg_index),
627  i,
628  ti.get_elem_type(),
629  col_heads[i],
630  row_count_heads[i],
631  (co_.device_type != ExecutorDeviceType::CPU || emit_only_preflight_fn)
632  ? nullptr
633  : input_str_dict_proxy_heads[i],
634  ctx,
635  cgen_state->ir_builder_);
636  func_args.push_back((pass_column_by_value
637  ? cgen_state->ir_builder_.CreateLoad(
638  col->getType()->getPointerElementType(), col)
639  : col_ptr));
640 
641  if (columnTypeRequiresCasting(ti) &&
643  columns_to_cast.push_back(std::make_pair(col_ptr, ti));
644  }
645  CHECK_EQ(col_index, -1);
646  } else if (ti.is_column_list()) {
647  if (col_index == -1) {
648  auto col_list = alloc_column_list(
649  std::string("input_col_list.") + std::to_string(func_arg_index),
650  ti.get_elem_type(),
651  col_heads[i],
652  ti.get_dimension(),
653  row_count_heads[i],
654  (emit_only_preflight_fn) ? nullptr : input_str_dict_proxy_heads[i],
655  ctx,
656  cgen_state->ir_builder_);
657  func_args.push_back(col_list);
658  }
659  col_index++;
660  if (col_index + 1 == ti.get_dimension()) {
661  col_index = -1;
662  }
663  } else {
664  throw std::runtime_error(
665  "Only integer and floating point columns or scalars are supported as inputs to "
666  "table "
667  "functions, got " +
668  ti.get_type_name());
669  }
670  }
671  auto output_str_dict_proxy_heads =
673  ? (generate_column_heads_load(exe_unit.target_exprs.size(),
674  output_str_dict_proxies_arg,
675  cgen_state->ir_builder_,
676  ctx))
677  : std::vector<llvm::Value*>();
678 
679  std::vector<llvm::Value*> output_col_args;
680  for (size_t i = 0; i < exe_unit.target_exprs.size(); i++) {
681  auto* gep = cgen_state->ir_builder_.CreateGEP(
682  output_buffers_arg->getType()->getScalarType()->getPointerElementType(),
683  output_buffers_arg,
684  cgen_state->llInt(i));
685  auto output_load =
686  cgen_state->ir_builder_.CreateLoad(gep->getType()->getPointerElementType(), gep);
687  const auto& expr = exe_unit.target_exprs[i];
688  const auto& ti = expr->get_type_info();
689  CHECK(!ti.is_column()); // UDTF output column type is its data type
690  CHECK(!ti.is_column_list()); // TODO: when UDTF outputs column_list, convert it to
691  // output columns
692  auto [col, col_ptr] = alloc_column(
693  std::string("output_col.") + std::to_string(i),
694  i,
695  ti,
697  ? output_load
698  : nullptr), // CPU: set_output_row_size will set the output
699  // Column ptr member
700  output_row_count_ptr,
701  co_.device_type == ExecutorDeviceType::CPU ? output_str_dict_proxy_heads[i]
702  : nullptr,
703  ctx,
704  cgen_state->ir_builder_);
705  if (co_.device_type == ExecutorDeviceType::CPU && !emit_only_preflight_fn) {
706  cgen_state->emitExternalCall(
707  "TableFunctionManager_register_output_column",
708  llvm::Type::getVoidTy(ctx),
709  {mgr_ptr, llvm::ConstantInt::get(get_int_type(32, ctx), i, true), col_ptr});
710  }
711  output_col_args.push_back((pass_column_by_value ? col : col_ptr));
712  }
713 
714  // output column members must be set before loading column when
715  // column instances are passed by value
716  if ((exe_unit.table_func.hasOutputSizeKnownPreLaunch() ||
717  exe_unit.table_func.hasPreFlightOutputSizer()) &&
718  (co_.device_type == ExecutorDeviceType::CPU) && !emit_only_preflight_fn) {
719  cgen_state->emitExternalCall(
720  "TableFunctionManager_set_output_row_size",
721  llvm::Type::getVoidTy(ctx),
722  {mgr_ptr,
723  cgen_state->ir_builder_.CreateLoad(
724  output_row_count_ptr->getType()->getPointerElementType(),
725  output_row_count_ptr)});
726  }
727 
728  if (!emit_only_preflight_fn) {
729  for (auto& col : output_col_args) {
730  func_args.push_back((pass_column_by_value
731  ? cgen_state->ir_builder_.CreateLoad(
732  col->getType()->getPointerElementType(), col)
733  : col));
734  }
735  }
736 
737  if (exe_unit.table_func.mayRequireCastingInputTypes() && !emit_only_preflight_fn) {
738  generateCastsForInputTypes(exe_unit, columns_to_cast, mgr_ptr);
739  }
740 
742  exe_unit, func_args, bb_exit, output_row_count_ptr, emit_only_preflight_fn);
743 
744  // std::cout << "=================================" << std::endl;
745  // entry_point_func_->print(llvm::outs());
746  // std::cout << "=================================" << std::endl;
747 
749 }
750 
752  const TableFunctionExecutionUnit& exe_unit,
753  const std::vector<std::pair<llvm::Value*, const SQLTypeInfo>>& columns_to_cast,
754  llvm::Value* mgr_ptr) {
755  auto* cgen_state = executor_->getCgenStatePtr();
756  llvm::LLVMContext& ctx = cgen_state->context_;
757  llvm::IRBuilder<>* ir_builder = &cgen_state->ir_builder_;
758  CodeGenerator codeGenerator = CodeGenerator(cgen_state, executor_->getPlanStatePtr());
759  llvm::Function* old_func = cgen_state->current_func_;
760  cgen_state->current_func_ =
761  entry_point_func_; // update cgen_state current func for CodeGenerator
762 
763  for (unsigned i = 0; i < columns_to_cast.size(); ++i) {
764  auto [col_ptr, ti] = columns_to_cast[i];
765 
766  if (ti.is_column() && ti.get_subtype() == kTIMESTAMP && ti.get_precision() != 9) {
767  // TIMESTAMP columns should always have nanosecond precision
768  SQLTypeInfo orig_ti = SQLTypeInfo(
769  ti.get_subtype(), ti.get_precision(), ti.get_dimension(), ti.get_notnull());
770  SQLTypeInfo dest_ti =
771  SQLTypeInfo(kTIMESTAMP, 9, ti.get_dimension(), ti.get_notnull());
772  cast_column(col_ptr,
774  orig_ti,
775  dest_ti,
776  std::to_string(i + 1),
777  *ir_builder,
778  ctx,
779  codeGenerator);
780  }
781  }
782 
783  // The QueryEngine CodeGenerator will codegen return values corresponding to
784  // QueryExecutionError codes. Since at the table function level we'd like error handling
785  // to be done by the TableFunctionManager, we replace the codegen'd returns by calls to
786  // the appropriate Manager functions.
788  // TableFunctionManager is not supported on GPU, so leave the QueryExecutionError code
789  return;
790  }
791 
792  std::vector<llvm::ReturnInst*> rets_to_replace;
793  for (llvm::BasicBlock& BB : *entry_point_func_) {
794  for (llvm::Instruction& I : BB) {
795  if (!llvm::isa<llvm::ReturnInst>(&I)) {
796  continue;
797  }
798  llvm::ReturnInst* RI = llvm::cast<llvm::ReturnInst>(&I);
799  llvm::Value* retValue = RI->getReturnValue();
800  if (!retValue || !llvm::isa<llvm::ConstantInt>(retValue)) {
801  continue;
802  }
803  llvm::ConstantInt* retConst = llvm::cast<llvm::ConstantInt>(retValue);
804  if (retConst->getValue() == 7) {
805  // ret 7 = underflow/overflow during casting attempt
806  rets_to_replace.push_back(RI);
807  }
808  }
809  }
810 
811  auto prev_insert_point = ir_builder->saveIP();
812  for (llvm::ReturnInst* RI : rets_to_replace) {
813  ir_builder->SetInsertPoint(RI);
814  llvm::Value* err_msg = ir_builder->CreateGlobalStringPtr(
815  "Underflow or overflow during casting of input types!", "cast_err_str");
816  llvm::Value* error_call;
817  if (exe_unit.table_func.usesManager()) {
818  error_call = cgen_state->emitExternalCall("TableFunctionManager_error_message",
819  ir_builder->getInt32Ty(),
820  {mgr_ptr, err_msg});
821  } else {
822  error_call = cgen_state->emitExternalCall(
823  "table_function_error", ir_builder->getInt32Ty(), {err_msg});
824  }
825  llvm::ReplaceInstWithInst(RI, llvm::ReturnInst::Create(ctx, error_call));
826  }
827  ir_builder->restoreIP(prev_insert_point);
828 
829  cgen_state->current_func_ = old_func;
830 }
831 
833  auto timer = DEBUG_TIMER(__func__);
835  std::vector<llvm::Type*> arg_types;
836  arg_types.reserve(entry_point_func_->arg_size());
837  std::for_each(entry_point_func_->arg_begin(),
838  entry_point_func_->arg_end(),
839  [&arg_types](const auto& arg) { arg_types.push_back(arg.getType()); });
840  CHECK_EQ(arg_types.size(), entry_point_func_->arg_size());
841 
842  auto cgen_state = executor_->getCgenStatePtr();
843  CHECK(cgen_state);
844  auto& ctx = cgen_state->context_;
845 
846  std::vector<llvm::Type*> wrapper_arg_types(arg_types.size() + 1);
847  wrapper_arg_types[0] = llvm::PointerType::get(get_int_type(32, ctx), 0);
848  wrapper_arg_types[1] = arg_types[0];
849 
850  for (size_t i = 1; i < arg_types.size(); ++i) {
851  wrapper_arg_types[i + 1] = arg_types[i];
852  }
853 
854  auto wrapper_ft =
855  llvm::FunctionType::get(llvm::Type::getVoidTy(ctx), wrapper_arg_types, false);
856  kernel_func_ = llvm::Function::Create(wrapper_ft,
857  llvm::Function::ExternalLinkage,
858  "table_func_kernel",
859  cgen_state->module_);
860 
861  auto wrapper_bb_entry = llvm::BasicBlock::Create(ctx, ".entry", kernel_func_, 0);
862  llvm::IRBuilder<> b(ctx);
863  b.SetInsertPoint(wrapper_bb_entry);
864  std::vector<llvm::Value*> loaded_args = {kernel_func_->arg_begin() + 1};
865  for (size_t i = 2; i < wrapper_arg_types.size(); ++i) {
866  loaded_args.push_back(kernel_func_->arg_begin() + i);
867  }
868  auto error_lv = b.CreateCall(entry_point_func_, loaded_args);
869  b.CreateStore(error_lv, kernel_func_->arg_begin());
870  b.CreateRetVoid();
871 }
872 
873 std::shared_ptr<CompilationContext> TableFunctionCompilationContext::finalize(
874  bool emit_only_preflight_fn) {
875  auto timer = DEBUG_TIMER(__func__);
876  /*
877  TODO 1: eliminate need for OverrideFromSrc
878  TODO 2: detect and link only the udf's that are needed
879  */
880  auto cgen_state = executor_->getCgenStatePtr();
881  auto is_gpu = co_.device_type == ExecutorDeviceType::GPU;
882  if (executor_->has_rt_udf_module(is_gpu)) {
883  CodeGenerator::link_udf_module(executor_->get_rt_udf_module(is_gpu),
884  *(cgen_state->module_),
885  cgen_state,
886  llvm::Linker::Flags::OverrideFromSrc);
887  }
888 
889  LOG(IR) << (emit_only_preflight_fn ? "Pre Flight Function Entry Point IR\n"
890  : "Table Function Entry Point IR\n")
892  std::shared_ptr<CompilationContext> code;
893  if (is_gpu) {
894  LOG(IR) << "Table Function Kernel IR\n" << serialize_llvm_object(kernel_func_);
895 
896  CHECK(executor_);
897  executor_->initializeNVPTXBackend();
898 
899  CodeGenerator::GPUTarget gpu_target{executor_->nvptx_target_machine_.get(),
900  executor_->cudaMgr(),
901  executor_->blockSize(),
902  cgen_state,
903  false};
906  kernel_func_,
908  /*is_gpu_smem_used=*/false,
909  co_,
910  gpu_target);
911  } else {
912  auto ee =
914  auto cpu_code = std::make_shared<CpuCompilationContext>(std::move(ee));
915  cpu_code->setFunctionPointer(entry_point_func_);
916  code = cpu_code;
917  }
918  LOG(IR) << "End of IR";
919 
920  return code;
921 }
llvm::Type * get_fp_ptr_type(const int width, llvm::LLVMContext &context)
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
#define CHECK_EQ(x, y)
Definition: Logger.h:231
std::string exprsKey(const std::vector< Analyzer::Expr * > &exprs)
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
bool is_timestamp() const
Definition: sqltypes.h:895
void generateEntryPoint(const TableFunctionExecutionUnit &exe_unit, bool emit_only_preflight_fn)
bool passColumnsByValue(const TableFunctionExecutionUnit &exe_unit)
std::vector< Analyzer::Expr * > input_exprs
const table_functions::TableFunction table_func
#define LOG(tag)
Definition: Logger.h:217
bool is_fp() const
Definition: sqltypes.h:514
std::tuple< llvm::Value *, llvm::Value * > alloc_column(std::string col_name, const size_t index, const SQLTypeInfo &data_target_info, llvm::Value *data_ptr, llvm::Value *data_size, llvm::Value *data_str_dict_proxy_ptr, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder)
llvm::Function * generate_entry_point(const CgenState *cgen_state)
std::shared_ptr< CompilationContext > finalize(bool emit_only_preflight_fn)
std::vector< std::string > CodeCacheKey
Definition: CodeCache.h:25
void generateCastsForInputTypes(const TableFunctionExecutionUnit &exe_unit, const std::vector< std::pair< llvm::Value *, const SQLTypeInfo >> &columns_to_cast, llvm::Value *mgr_ptr)
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
llvm::Value * alloc_column_list(std::string col_list_name, const SQLTypeInfo &data_target_info, llvm::Value *data_ptrs, int length, llvm::Value *data_size, llvm::Value *data_str_dict_proxy_ptrs, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder)
std::string to_string(char const *&&v)
llvm::Module * module_
Definition: CgenState.h:350
void verify_function_ir(const llvm::Function *func)
size_t get_bit_width(const SQLTypeInfo &ti)
llvm::LLVMContext & context_
Definition: CgenState.h:359
bool is_integer() const
Definition: sqltypes.h:512
bool is_boolean() const
Definition: sqltypes.h:517
static void link_udf_module(const std::unique_ptr< llvm::Module > &udf_module, llvm::Module &module, CgenState *cgen_state, llvm::Linker::Flags flags=llvm::Linker::Flags::None)
void generateTableFunctionCall(const TableFunctionExecutionUnit &exe_unit, const std::vector< llvm::Value * > &func_args, llvm::BasicBlock *bb_exit, llvm::Value *output_row_count_ptr, bool emit_only_preflight_fn)
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1453
std::string getName(const bool drop_suffix=false, const bool lower=false) const
int get_precision() const
Definition: sqltypes.h:332
ExecutorDeviceType device_type
bool is_column() const
Definition: sqltypes.h:523
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
std::string serialize_llvm_object(const T *llvm_obj)
static std::shared_ptr< GpuCompilationContext > generateNativeGPUCode(Executor *executor, llvm::Function *func, llvm::Function *wrapper_func, const std::unordered_set< llvm::Function * > &live_funcs, const bool is_gpu_smem_used, const CompilationOptions &co, const GPUTarget &gpu_target)
static CodeCacheAccessor< CompilationContext > tf_code_accessor
Definition: Execute.h:1273
std::vector< llvm::Value * > generate_column_heads_load(const int num_columns, llvm::Value *byte_stream_arg, llvm::IRBuilder<> &ir_builder, llvm::LLVMContext &ctx)
std::string get_type_name() const
Definition: sqltypes.h:443
void cast_column(llvm::Value *col_base_ptr, llvm::Function *func, SQLTypeInfo &orig_ti, SQLTypeInfo &dest_ti, std::string index, llvm::IRBuilder<> &ir_builder, llvm::LLVMContext &ctx, CodeGenerator &codeGenerator)
llvm::Value * codegenCastBetweenTimestamps(llvm::Value *ts_lv, const SQLTypeInfo &operand_dimen, const SQLTypeInfo &target_dimen, const bool nullable)
Definition: CastIR.cpp:170
llvm::Type * get_llvm_type_from_sql_column_type(const SQLTypeInfo elem_ti, llvm::LLVMContext &ctx)
bool is_bytes() const
Definition: sqltypes.h:525
#define CHECK(condition)
Definition: Logger.h:223
#define DEBUG_TIMER(name)
Definition: Logger.h:370
std::shared_ptr< CompilationContext > compile(const TableFunctionExecutionUnit &exe_unit, bool emit_only_preflight_fn)
std::vector< Analyzer::Expr * > target_exprs
bool is_string() const
Definition: sqltypes.h:510
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336
llvm::Type * get_int_ptr_type(const int width, llvm::LLVMContext &context)
llvm::Value * cast_value(llvm::Value *value, SQLTypeInfo &orig_ti, SQLTypeInfo &dest_ti, bool nullable, CodeGenerator &codeGenerator)