OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
TableFunctionCompilationContext.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #include <llvm/IR/Verifier.h>
20 #include <llvm/Support/raw_os_ostream.h>
21 #include <algorithm>
22 #include <boost/algorithm/string.hpp>
23 
25 
26 extern std::unique_ptr<llvm::Module> g_rt_module;
27 extern std::unique_ptr<llvm::Module> rt_udf_cpu_module;
28 extern std::unique_ptr<llvm::Module> rt_udf_gpu_module;
29 
30 namespace {
31 
32 llvm::Function* generate_entry_point(const CgenState* cgen_state) {
33  auto& ctx = cgen_state->context_;
34  const auto pi8_type = llvm::PointerType::get(get_int_type(8, ctx), 0);
35  const auto ppi8_type = llvm::PointerType::get(pi8_type, 0);
36  const auto pi64_type = llvm::PointerType::get(get_int_type(64, ctx), 0);
37  const auto ppi64_type = llvm::PointerType::get(pi64_type, 0);
38  const auto i32_type = get_int_type(32, ctx);
39 
40  const auto func_type = llvm::FunctionType::get(
41  i32_type, {ppi8_type, pi64_type, ppi64_type, pi64_type}, false);
42 
43  auto func = llvm::Function::Create(func_type,
44  llvm::Function::ExternalLinkage,
45  "call_table_function",
46  cgen_state->module_);
47  auto arg_it = func->arg_begin();
48  const auto input_cols_arg = &*arg_it;
49  input_cols_arg->setName("input_col_buffers");
50  const auto input_row_counts = &*(++arg_it);
51  input_row_counts->setName("input_row_counts");
52  const auto output_buffers = &*(++arg_it);
53  output_buffers->setName("output_buffers");
54  const auto output_row_count = &*(++arg_it);
55  output_row_count->setName("output_row_count");
56  return func;
57 }
58 
60  llvm::LLVMContext& ctx) {
61  if (elem_ti.is_fp()) {
62  switch (elem_ti.get_size()) {
63  case 4:
64  return llvm::Type::getFloatPtrTy(ctx);
65  case 8:
66  return llvm::Type::getDoublePtrTy(ctx);
67  }
68  }
69  if (elem_ti.is_boolean()) {
70  return llvm::Type::getInt8PtrTy(ctx);
71  }
72  CHECK(elem_ti.is_integer());
73  switch (elem_ti.get_size()) {
74  case 1:
75  return llvm::Type::getInt8PtrTy(ctx);
76  case 2:
77  return llvm::Type::getInt16PtrTy(ctx);
78  case 4:
79  return llvm::Type::getInt32PtrTy(ctx);
80  case 8:
81  return llvm::Type::getInt64PtrTy(ctx);
82  }
83  LOG(FATAL) << "get_llvm_type_from_sql_column_type: not implemented for "
84  << ::toString(elem_ti);
85  return nullptr;
86 }
87 
88 llvm::Value* alloc_column(std::string col_name,
89  const SQLTypeInfo& data_target_info,
90  llvm::Value* data_ptr,
91  llvm::Value* data_size,
92  llvm::LLVMContext& ctx,
93  llvm::IRBuilder<>& ir_builder,
94  bool byval) {
95  /*
96  Creates a new Column instance of given element type and initialize
97  its data ptr and sz members. If data ptr or sz are unspecified
98  (have nullptr values) then the corresponding members are
99  initialized with NULL and -1, respectively.
100  */
101  llvm::Type* data_ptr_llvm_type =
102  get_llvm_type_from_sql_column_type(data_target_info, ctx);
103  llvm::StructType* col_struct_type =
104  llvm::StructType::get(ctx,
105  {
106  data_ptr_llvm_type, /* T* ptr */
107  llvm::Type::getInt64Ty(ctx) /* int64_t sz */
108  });
109  auto col = ir_builder.CreateAlloca(col_struct_type);
110  col->setName(col_name);
111  auto col_ptr_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 0);
112  auto col_sz_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 1);
113  col_ptr_ptr->setName(col_name + ".ptr");
114  col_sz_ptr->setName(col_name + ".sz");
115 
116  if (data_ptr != nullptr) {
117  if (data_ptr->getType() == data_ptr_llvm_type->getPointerElementType()) {
118  ir_builder.CreateStore(data_ptr, col_ptr_ptr);
119  } else {
120  auto tmp = ir_builder.CreateBitCast(data_ptr, data_ptr_llvm_type);
121  ir_builder.CreateStore(tmp, col_ptr_ptr);
122  }
123  } else {
124  ir_builder.CreateStore(llvm::Constant::getNullValue(data_ptr_llvm_type), col_ptr_ptr);
125  }
126  if (data_size != nullptr) {
127  auto data_size_type = data_size->getType();
128  if (data_size_type->isPointerTy()) {
129  CHECK(data_size_type->getPointerElementType()->isIntegerTy(64));
130  auto val = ir_builder.CreateLoad(data_size);
131  ir_builder.CreateStore(val, col_sz_ptr);
132  } else {
133  CHECK(data_size_type->isIntegerTy(64));
134  ir_builder.CreateStore(data_size, col_sz_ptr);
135  }
136  } else {
137  auto const_minus1 = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), -1, true);
138  ir_builder.CreateStore(const_minus1, col_sz_ptr);
139  }
140 
141  if (byval) {
142  return ir_builder.CreateLoad(col);
143  } else {
144  auto col_ptr = ir_builder.CreatePointerCast(
145  col_ptr_ptr, llvm::PointerType::get(llvm::Type::getInt8Ty(ctx), 0));
146  col_ptr->setName(col_name + "_ptr");
147  return col_ptr;
148  }
149 }
150 
151 llvm::Value* alloc_column_list(std::string col_list_name,
152  const SQLTypeInfo& data_target_info,
153  llvm::Value* data_ptrs,
154  int length,
155  llvm::Value* data_size,
156  llvm::LLVMContext& ctx,
157  llvm::IRBuilder<>& ir_builder) {
158  /*
159  Creates a new ColumnList instance of given element type and initialize
160  its members. If data ptr or size are unspecified (have nullptr
161  values) then the corresponding members are initialized with NULL
162  and -1, respectively.
163  */
164  llvm::Type* data_ptrs_llvm_type = llvm::Type::getInt8PtrTy(ctx);
165 
166  llvm::StructType* col_list_struct_type =
167  llvm::StructType::get(ctx,
168  {
169  data_ptrs_llvm_type, /* int8_t* ptrs */
170  llvm::Type::getInt64Ty(ctx), /* int64_t length */
171  llvm::Type::getInt64Ty(ctx) /* int64_t size */
172  });
173  auto col_list = ir_builder.CreateAlloca(col_list_struct_type);
174  col_list->setName(col_list_name);
175  auto col_list_ptr_ptr = ir_builder.CreateStructGEP(col_list_struct_type, col_list, 0);
176  auto col_list_length_ptr =
177  ir_builder.CreateStructGEP(col_list_struct_type, col_list, 1);
178  auto col_list_size_ptr = ir_builder.CreateStructGEP(col_list_struct_type, col_list, 2);
179 
180  col_list_ptr_ptr->setName(col_list_name + ".ptrs");
181  col_list_length_ptr->setName(col_list_name + ".length");
182  col_list_size_ptr->setName(col_list_name + ".size");
183 
184  CHECK(length >= 0);
185  auto const_length = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), length, true);
186 
187  if (data_ptrs != nullptr) {
188  if (data_ptrs->getType() == data_ptrs_llvm_type->getPointerElementType()) {
189  ir_builder.CreateStore(data_ptrs, col_list_ptr_ptr);
190  } else {
191  auto tmp = ir_builder.CreateBitCast(data_ptrs, data_ptrs_llvm_type);
192  ir_builder.CreateStore(tmp, col_list_ptr_ptr);
193  }
194  } else {
195  ir_builder.CreateStore(llvm::Constant::getNullValue(data_ptrs_llvm_type),
196  col_list_ptr_ptr);
197  }
198 
199  ir_builder.CreateStore(const_length, col_list_length_ptr);
200 
201  if (data_size != nullptr) {
202  auto data_size_type = data_size->getType();
203  if (data_size_type->isPointerTy()) {
204  CHECK(data_size_type->getPointerElementType()->isIntegerTy(64));
205  auto size_val = ir_builder.CreateLoad(data_size);
206  ir_builder.CreateStore(size_val, col_list_size_ptr);
207  } else {
208  CHECK(data_size_type->isIntegerTy(64));
209  ir_builder.CreateStore(data_size, col_list_size_ptr);
210  }
211  } else {
212  auto const_minus1 = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), -1, true);
213  ir_builder.CreateStore(const_minus1, col_list_size_ptr);
214  }
215 
216  auto col_list_ptr = ir_builder.CreatePointerCast(
217  col_list_ptr_ptr, llvm::PointerType::get(llvm::Type::getInt8Ty(ctx), 0));
218  col_list_ptr->setName(col_list_name + "_ptrs");
219  return col_list_ptr;
220 }
221 
222 } // namespace
223 
225  : cgen_state_(std::make_unique<CgenState>(/*num_query_infos=*/0,
226  /*contains_left_deep_outer_join=*/false)) {
227  auto cgen_state = cgen_state_.get();
228  CHECK(cgen_state);
229 
230  std::unique_ptr<llvm::Module> module(runtime_module_shallow_copy(cgen_state));
231  cgen_state->module_ = module.get();
232 
234  module_ = std::move(module);
235 }
236 
238  const CompilationOptions& co,
239  Executor* executor) {
240  generateEntryPoint(exe_unit);
243  }
244  finalize(co, executor);
245 }
246 
248  const TableFunctionExecutionUnit& exe_unit) {
250  auto arg_it = entry_point_func_->arg_begin();
251  const auto input_cols_arg = &*arg_it;
252  const auto input_row_counts_arg = &*(++arg_it);
253  const auto output_buffers_arg = &*(++arg_it);
254  const auto output_row_count_ptr = &*(++arg_it);
255 
256  auto cgen_state = cgen_state_.get();
257  CHECK(cgen_state);
258  auto& ctx = cgen_state->context_;
259 
260  const auto bb_entry = llvm::BasicBlock::Create(ctx, ".entry", entry_point_func_, 0);
261  cgen_state->ir_builder_.SetInsertPoint(bb_entry);
262 
263  const auto bb_exit = llvm::BasicBlock::Create(ctx, ".exit", entry_point_func_);
264 
265  const auto func_body_bb = llvm::BasicBlock::Create(
266  ctx, ".func_body", cgen_state->ir_builder_.GetInsertBlock()->getParent());
267  cgen_state->ir_builder_.SetInsertPoint(func_body_bb);
268 
269  auto col_heads = generate_column_heads_load(
270  exe_unit.input_exprs.size(), input_cols_arg, cgen_state->ir_builder_, ctx);
271  CHECK_EQ(exe_unit.input_exprs.size(), col_heads.size());
272 
273  auto row_count_heads = generate_column_heads_load(
274  exe_unit.input_exprs.size(), input_row_counts_arg, cgen_state->ir_builder_, ctx);
275 
276  // The column arguments of C++ UDTFs processed by clang must be
277  // passed by reference, see rbc issues 200 and 289.
278  auto pass_column_by_value = exe_unit.table_func.isRuntime();
279  std::vector<llvm::Value*> func_args;
280  size_t func_arg_index = 0;
281  int col_index = -1;
282  for (size_t i = 0; i < exe_unit.input_exprs.size(); i++) {
283  const auto& expr = exe_unit.input_exprs[i];
284  const auto& ti = expr->get_type_info();
285  if (col_index == -1) {
286  func_arg_index += 1;
287  }
288  if (ti.is_fp()) {
289  auto r = cgen_state->ir_builder_.CreateBitCast(
290  col_heads[i], llvm::PointerType::get(get_fp_type(get_bit_width(ti), ctx), 0));
291  func_args.push_back(cgen_state->ir_builder_.CreateLoad(r));
292  CHECK_EQ(col_index, -1);
293  } else if (ti.is_integer()) {
294  auto r = cgen_state->ir_builder_.CreateBitCast(
295  col_heads[i], llvm::PointerType::get(get_int_type(get_bit_width(ti), ctx), 0));
296  func_args.push_back(cgen_state->ir_builder_.CreateLoad(r));
297  CHECK_EQ(col_index, -1);
298  } else if (ti.is_column()) {
299  auto col = alloc_column(std::string("input_col.") + std::to_string(func_arg_index),
300  ti.get_elem_type(),
301  col_heads[i],
302  row_count_heads[i],
303  ctx,
304  cgen_state_->ir_builder_,
305  pass_column_by_value);
306  func_args.push_back(col);
307  CHECK_EQ(col_index, -1);
308  } else if (ti.is_column_list()) {
309  if (col_index == -1) {
310  auto col_list = alloc_column_list(
311  std::string("input_col_list.") + std::to_string(func_arg_index),
312  ti.get_elem_type(),
313  col_heads[i],
314  ti.get_dimension(),
315  row_count_heads[i],
316  ctx,
317  cgen_state_->ir_builder_);
318  func_args.push_back(col_list);
319  }
320  col_index++;
321  if (col_index + 1 == ti.get_dimension()) {
322  col_index = -1;
323  }
324  } else {
325  throw std::runtime_error(
326  "Only integer and floating point columns or scalars are supported as inputs to "
327  "table "
328  "functions, got " +
329  ti.get_type_name());
330  }
331  }
332  std::vector<llvm::Value*> output_col_args;
333  for (size_t i = 0; i < exe_unit.target_exprs.size(); i++) {
334  auto output_load = cgen_state->ir_builder_.CreateLoad(
335  cgen_state->ir_builder_.CreateGEP(output_buffers_arg, cgen_state_->llInt(i)));
336  const auto& expr = exe_unit.target_exprs[i];
337  const auto& ti = expr->get_type_info();
338  CHECK(!ti.is_column()); // UDTF output column type is its data type
339  CHECK(!ti.is_column_list()); // TODO: when UDTF outputs column_list, convert it to
340  // output columns
341  auto col = alloc_column(std::string("output_col.") + std::to_string(i),
342  ti,
343  output_load,
344  output_row_count_ptr,
345  ctx,
346  cgen_state_->ir_builder_,
347  pass_column_by_value);
348  func_args.push_back(col);
349  }
350  auto func_name = exe_unit.table_func.getName();
351  boost::algorithm::to_lower(func_name);
352  const auto table_func_return =
353  cgen_state->emitExternalCall(func_name, get_int_type(32, ctx), func_args);
354  table_func_return->setName("table_func_ret");
355 
356  // If table_func_return is non-negative then store the value in
357  // output_row_count and return zero. Otherwise, return
358  // table_func_return that negative value contains the error code.
359  const auto bb_exit_0 = llvm::BasicBlock::Create(ctx, ".exit0", entry_point_func_);
360 
361  auto const_zero = llvm::ConstantInt::get(table_func_return->getType(), 0, true);
362  auto is_ok = cgen_state_->ir_builder_.CreateICmpSGE(table_func_return, const_zero);
363  cgen_state_->ir_builder_.CreateCondBr(is_ok, bb_exit_0, bb_exit);
364 
365  cgen_state_->ir_builder_.SetInsertPoint(bb_exit_0);
366  auto r = cgen_state->ir_builder_.CreateIntCast(
367  table_func_return, get_int_type(64, ctx), true);
368  cgen_state->ir_builder_.CreateStore(r, output_row_count_ptr);
369  cgen_state->ir_builder_.CreateRet(const_zero);
370 
371  cgen_state->ir_builder_.SetInsertPoint(bb_exit);
372  cgen_state->ir_builder_.CreateRet(table_func_return);
373 
374  cgen_state->ir_builder_.SetInsertPoint(bb_entry);
375  cgen_state->ir_builder_.CreateBr(func_body_bb);
376 
377  /*
378  std::cout << "=================================" << std::endl;
379  entry_point_func_->print(llvm::outs());
380  std::cout << "=================================" << std::endl;
381  */
382 
384 }
385 
388  std::vector<llvm::Type*> arg_types;
389  arg_types.reserve(entry_point_func_->arg_size());
390  std::for_each(entry_point_func_->arg_begin(),
391  entry_point_func_->arg_end(),
392  [&arg_types](const auto& arg) { arg_types.push_back(arg.getType()); });
393  CHECK_EQ(arg_types.size(), entry_point_func_->arg_size());
394 
395  auto cgen_state = cgen_state_.get();
396  CHECK(cgen_state);
397  auto& ctx = cgen_state->context_;
398 
399  std::vector<llvm::Type*> wrapper_arg_types(arg_types.size() + 1);
400  wrapper_arg_types[0] = llvm::PointerType::get(get_int_type(32, ctx), 0);
401  wrapper_arg_types[1] = arg_types[0];
402 
403  for (size_t i = 1; i < arg_types.size(); ++i) {
404  wrapper_arg_types[i + 1] = arg_types[i];
405  }
406 
407  auto wrapper_ft =
408  llvm::FunctionType::get(llvm::Type::getVoidTy(ctx), wrapper_arg_types, false);
409  kernel_func_ = llvm::Function::Create(wrapper_ft,
410  llvm::Function::ExternalLinkage,
411  "table_func_kernel",
412  cgen_state->module_);
413 
414  auto wrapper_bb_entry = llvm::BasicBlock::Create(ctx, ".entry", kernel_func_, 0);
415  llvm::IRBuilder<> b(ctx);
416  b.SetInsertPoint(wrapper_bb_entry);
417  std::vector<llvm::Value*> loaded_args = {kernel_func_->arg_begin() + 1};
418  for (size_t i = 2; i < wrapper_arg_types.size(); ++i) {
419  loaded_args.push_back(kernel_func_->arg_begin() + i);
420  }
421  auto error_lv = b.CreateCall(entry_point_func_, loaded_args);
422  b.CreateStore(error_lv, kernel_func_->arg_begin());
423  b.CreateRetVoid();
424 }
425 
427  Executor* executor) {
428  /*
429  TODO 1: eliminate need for OverrideFromSrc
430  TODO 2: detect and link only the udf's that are needed
431  */
432  if (co.device_type == ExecutorDeviceType::GPU && rt_udf_gpu_module != nullptr) {
434  *module_,
435  cgen_state_.get(),
436  llvm::Linker::Flags::OverrideFromSrc);
437  }
438  if (co.device_type == ExecutorDeviceType::CPU && rt_udf_cpu_module != nullptr) {
440  *module_,
441  cgen_state_.get(),
442  llvm::Linker::Flags::OverrideFromSrc);
443  }
444 
445  module_.release();
446  // Add code to cache?
447 
448  LOG(IR) << "Table Function Entry Point IR\n"
450 
452  LOG(IR) << "Table Function Kernel IR\n" << serialize_llvm_object(kernel_func_);
453 
454  CHECK(executor);
455  executor->initializeNVPTXBackend();
456  const auto cuda_mgr = executor->catalog_->getDataMgr().getCudaMgr();
457  CHECK(cuda_mgr);
458 
459  CodeGenerator::GPUTarget gpu_target{executor->nvptx_target_machine_.get(),
460  cuda_mgr,
461  executor->blockSize(),
462  cgen_state_.get(),
463  false};
465  kernel_func_,
467  co,
468  gpu_target);
469  } else {
470  auto ee =
472  func_ptr = reinterpret_cast<FuncPtr>(ee->getPointerToFunction(entry_point_func_));
473  own_execution_engine_ = std::move(ee);
474  }
475 
476  LOG(IR) << "End of IR";
477 }
std::string to_lower(const std::string &str)
#define CHECK_EQ(x, y)
Definition: Logger.h:211
std::unique_ptr< llvm::Module > rt_udf_cpu_module
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
std::string toString(const ExtArgumentType &sig_type)
std::unique_ptr< llvm::Module > runtime_module_shallow_copy(CgenState *cgen_state)
std::vector< Analyzer::Expr * > input_exprs
const table_functions::TableFunction table_func
void generateEntryPoint(const TableFunctionExecutionUnit &exe_unit)
#define LOG(tag)
Definition: Logger.h:194
std::unique_ptr< llvm::Module > rt_udf_gpu_module
bool is_fp() const
Definition: sqltypes.h:493
std::shared_ptr< GpuCompilationContext > gpu_code_
tuple r
Definition: test_fsi.py:16
llvm::Function * generate_entry_point(const CgenState *cgen_state)
int32_t(*)(const int8_t **input_cols, const int64_t *input_row_count, int64_t **out, int64_t *output_row_count) FuncPtr
std::unique_ptr< llvm::Module > module_
llvm::Type * get_fp_type(const int width, llvm::LLVMContext &context)
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
std::string to_string(char const *&&v)
static std::shared_ptr< GpuCompilationContext > generateNativeGPUCode(llvm::Function *func, llvm::Function *wrapper_func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co, const GPUTarget &gpu_target)
llvm::Module * module_
Definition: CgenState.h:324
void verify_function_ir(const llvm::Function *func)
size_t get_bit_width(const SQLTypeInfo &ti)
llvm::LLVMContext & context_
Definition: CgenState.h:333
bool is_integer() const
Definition: sqltypes.h:491
std::unique_ptr< llvm::Module > g_rt_module
llvm::Value * alloc_column(std::string col_name, const SQLTypeInfo &data_target_info, llvm::Value *data_ptr, llvm::Value *data_size, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder, bool byval)
bool is_boolean() const
Definition: sqltypes.h:496
static void link_udf_module(const std::unique_ptr< llvm::Module > &udf_module, llvm::Module &module, CgenState *cgen_state, llvm::Linker::Flags flags=llvm::Linker::Flags::None)
std::string getName(const bool drop_suffix=false, const bool lower=false) const
ExecutorDeviceType device_type
void finalize(const CompilationOptions &co, Executor *executor)
std::string serialize_llvm_object(const T *llvm_obj)
std::vector< llvm::Value * > generate_column_heads_load(const int num_columns, llvm::Value *byte_stream_arg, llvm::IRBuilder<> &ir_builder, llvm::LLVMContext &ctx)
llvm::Value * alloc_column_list(std::string col_list_name, const SQLTypeInfo &data_target_info, llvm::Value *data_ptrs, int length, llvm::Value *data_size, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder)
llvm::Type * get_llvm_type_from_sql_column_type(const SQLTypeInfo elem_ti, llvm::LLVMContext &ctx)
bool g_enable_watchdog false
Definition: Execute.cpp:76
#define CHECK(condition)
Definition: Logger.h:203
std::vector< Analyzer::Expr * > target_exprs
void compile(const TableFunctionExecutionUnit &exe_unit, const CompilationOptions &co, Executor *executor)