19 #include <llvm/IR/Verifier.h>
20 #include <llvm/Support/raw_os_ostream.h>
22 #include <boost/algorithm/string.hpp>
34 const auto pi8_type = llvm::PointerType::get(
get_int_type(8, ctx), 0);
35 const auto ppi8_type = llvm::PointerType::get(pi8_type, 0);
36 const auto pi64_type = llvm::PointerType::get(
get_int_type(64, ctx), 0);
37 const auto ppi64_type = llvm::PointerType::get(pi64_type, 0);
40 const auto func_type = llvm::FunctionType::get(
41 i32_type, {ppi8_type, pi64_type, ppi64_type, pi64_type},
false);
43 auto func = llvm::Function::Create(func_type,
44 llvm::Function::ExternalLinkage,
45 "call_table_function",
47 auto arg_it = func->arg_begin();
48 const auto input_cols_arg = &*arg_it;
49 input_cols_arg->setName(
"input_col_buffers");
50 const auto input_row_counts = &*(++arg_it);
51 input_row_counts->setName(
"input_row_counts");
52 const auto output_buffers = &*(++arg_it);
53 output_buffers->setName(
"output_buffers");
54 const auto output_row_count = &*(++arg_it);
55 output_row_count->setName(
"output_row_count");
60 llvm::LLVMContext& ctx) {
61 if (elem_ti.
is_fp()) {
64 return llvm::Type::getFloatPtrTy(ctx);
66 return llvm::Type::getDoublePtrTy(ctx);
70 return llvm::Type::getInt8PtrTy(ctx);
75 return llvm::Type::getInt8PtrTy(ctx);
77 return llvm::Type::getInt16PtrTy(ctx);
79 return llvm::Type::getInt32PtrTy(ctx);
81 return llvm::Type::getInt64PtrTy(ctx);
83 LOG(
FATAL) <<
"get_llvm_type_from_sql_column_type: not implemented for "
90 llvm::Value* data_ptr,
91 llvm::Value* data_size,
92 llvm::LLVMContext& ctx,
93 llvm::IRBuilder<>& ir_builder,
103 llvm::StructType* col_struct_type =
104 llvm::StructType::get(ctx,
107 llvm::Type::getInt64Ty(ctx)
109 auto col = ir_builder.CreateAlloca(col_struct_type);
110 col->setName(col_name);
111 auto col_ptr_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 0);
112 auto col_sz_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 1);
113 col_ptr_ptr->setName(col_name +
".ptr");
114 col_sz_ptr->setName(col_name +
".sz");
116 if (data_ptr !=
nullptr) {
117 if (data_ptr->getType() == data_ptr_llvm_type->getPointerElementType()) {
118 ir_builder.CreateStore(data_ptr, col_ptr_ptr);
120 auto tmp = ir_builder.CreateBitCast(data_ptr, data_ptr_llvm_type);
121 ir_builder.CreateStore(tmp, col_ptr_ptr);
124 ir_builder.CreateStore(llvm::Constant::getNullValue(data_ptr_llvm_type), col_ptr_ptr);
126 if (data_size !=
nullptr) {
127 auto data_size_type = data_size->getType();
128 if (data_size_type->isPointerTy()) {
129 CHECK(data_size_type->getPointerElementType()->isIntegerTy(64));
130 auto val = ir_builder.CreateLoad(data_size);
131 ir_builder.CreateStore(val, col_sz_ptr);
133 CHECK(data_size_type->isIntegerTy(64));
134 ir_builder.CreateStore(data_size, col_sz_ptr);
137 auto const_minus1 = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), -1,
true);
138 ir_builder.CreateStore(const_minus1, col_sz_ptr);
142 return ir_builder.CreateLoad(col);
144 auto col_ptr = ir_builder.CreatePointerCast(
145 col_ptr_ptr, llvm::PointerType::get(llvm::Type::getInt8Ty(ctx), 0));
146 col_ptr->setName(col_name +
"_ptr");
154 : cgen_state_(std::make_unique<
CgenState>(0,
160 cgen_state->module_ = module.get();
168 Executor* executor) {
180 const auto input_cols_arg = &*arg_it;
181 const auto input_row_counts_arg = &*(++arg_it);
182 const auto output_buffers_arg = &*(++arg_it);
183 const auto output_row_count_ptr = &*(++arg_it);
187 auto& ctx = cgen_state->context_;
189 const auto bb_entry = llvm::BasicBlock::Create(ctx,
".entry",
entry_point_func_, 0);
190 cgen_state->ir_builder_.SetInsertPoint(bb_entry);
194 const auto func_body_bb = llvm::BasicBlock::Create(
195 ctx,
".func_body", cgen_state->ir_builder_.GetInsertBlock()->getParent());
196 cgen_state->ir_builder_.SetInsertPoint(func_body_bb);
199 exe_unit.
input_exprs.size(), input_cols_arg, cgen_state->ir_builder_, ctx);
203 exe_unit.
input_exprs.size(), input_row_counts_arg, cgen_state->ir_builder_, ctx);
208 std::vector<llvm::Value*> func_args;
211 const auto& ti = expr->get_type_info();
213 auto r = cgen_state->ir_builder_.CreateBitCast(
215 func_args.push_back(cgen_state->ir_builder_.CreateLoad(
r));
216 }
else if (ti.is_integer()) {
217 auto r = cgen_state->ir_builder_.CreateBitCast(
219 func_args.push_back(cgen_state->ir_builder_.CreateLoad(
r));
220 }
else if (ti.is_column()) {
227 pass_column_by_value);
228 func_args.push_back(col);
230 throw std::runtime_error(
231 "Only integer and floating point columns or scalars are supported as inputs to "
237 std::vector<llvm::Value*> output_col_args;
239 auto output_load = cgen_state->ir_builder_.CreateLoad(
240 cgen_state->ir_builder_.CreateGEP(output_buffers_arg,
cgen_state_->llInt(
i)));
242 const auto& ti = expr->get_type_info();
243 CHECK(!ti.is_column());
247 output_row_count_ptr,
250 pass_column_by_value);
251 func_args.push_back(col);
255 const auto table_func_return =
256 cgen_state->emitExternalCall(func_name,
get_int_type(32, ctx), func_args);
257 table_func_return->setName(
"table_func_ret");
262 const auto bb_exit_0 = llvm::BasicBlock::Create(ctx,
".exit0",
entry_point_func_);
264 auto const_zero = llvm::ConstantInt::get(table_func_return->getType(), 0,
true);
265 auto is_ok =
cgen_state_->ir_builder_.CreateICmpSGE(table_func_return, const_zero);
266 cgen_state_->ir_builder_.CreateCondBr(is_ok, bb_exit_0, bb_exit);
268 cgen_state_->ir_builder_.SetInsertPoint(bb_exit_0);
269 auto r = cgen_state->ir_builder_.CreateIntCast(
271 cgen_state->ir_builder_.CreateStore(
r, output_row_count_ptr);
272 cgen_state->ir_builder_.CreateRet(const_zero);
274 cgen_state->ir_builder_.SetInsertPoint(bb_exit);
275 cgen_state->ir_builder_.CreateRet(table_func_return);
277 cgen_state->ir_builder_.SetInsertPoint(bb_entry);
278 cgen_state->ir_builder_.CreateBr(func_body_bb);
291 std::vector<llvm::Type*> arg_types;
295 [&arg_types](
const auto& arg) { arg_types.push_back(arg.getType()); });
300 auto& ctx = cgen_state->context_;
302 std::vector<llvm::Type*> wrapper_arg_types(arg_types.size() + 1);
303 wrapper_arg_types[0] = llvm::PointerType::get(
get_int_type(32, ctx), 0);
304 wrapper_arg_types[1] = arg_types[0];
306 for (
size_t i = 1;
i < arg_types.size(); ++
i) {
307 wrapper_arg_types[
i + 1] = arg_types[
i];
311 llvm::FunctionType::get(llvm::Type::getVoidTy(ctx), wrapper_arg_types,
false);
313 llvm::Function::ExternalLinkage,
315 cgen_state->module_);
317 auto wrapper_bb_entry = llvm::BasicBlock::Create(ctx,
".entry",
kernel_func_, 0);
318 llvm::IRBuilder<> b(ctx);
319 b.SetInsertPoint(wrapper_bb_entry);
320 std::vector<llvm::Value*> loaded_args = {
kernel_func_->arg_begin() + 1};
321 for (
size_t i = 2;
i < wrapper_arg_types.size(); ++
i) {
330 Executor* executor) {
339 llvm::Linker::Flags::OverrideFromSrc);
345 llvm::Linker::Flags::OverrideFromSrc);
351 LOG(
IR) <<
"Table Function Entry Point IR\n"
358 executor->initializeNVPTXBackend();
359 const auto cuda_mgr = executor->catalog_->getDataMgr().getCudaMgr();
364 executor->blockSize(),
379 LOG(
IR) <<
"End of IR";
llvm::Function * kernel_func_
std::unique_ptr< llvm::Module > rt_udf_cpu_module
HOST DEVICE int get_size() const
std::unique_ptr< CgenState > cgen_state_
TableFunctionCompilationContext()
std::string toString(const ExtArgumentType &sig_type)
std::unique_ptr< llvm::Module > runtime_module_shallow_copy(CgenState *cgen_state)
std::vector< Analyzer::Expr * > input_exprs
const table_functions::TableFunction table_func
void generateEntryPoint(const TableFunctionExecutionUnit &exe_unit)
std::unique_ptr< llvm::Module > rt_udf_gpu_module
std::shared_ptr< GpuCompilationContext > gpu_code_
llvm::Function * generate_entry_point(const CgenState *cgen_state)
int32_t(*)(const int8_t **input_cols, const int64_t *input_row_count, int64_t **out, int64_t *output_row_count) FuncPtr
std::unique_ptr< llvm::Module > module_
llvm::Type * get_fp_type(const int width, llvm::LLVMContext &context)
ExecutionEngineWrapper own_execution_engine_
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
static std::shared_ptr< GpuCompilationContext > generateNativeGPUCode(llvm::Function *func, llvm::Function *wrapper_func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co, const GPUTarget &gpu_target)
void verify_function_ir(const llvm::Function *func)
size_t get_bit_width(const SQLTypeInfo &ti)
llvm::LLVMContext & context_
std::unique_ptr< llvm::Module > g_rt_module
llvm::Value * alloc_column(std::string col_name, const SQLTypeInfo &data_target_info, llvm::Value *data_ptr, llvm::Value *data_size, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder, bool byval)
static void link_udf_module(const std::unique_ptr< llvm::Module > &udf_module, llvm::Module &module, CgenState *cgen_state, llvm::Linker::Flags flags=llvm::Linker::Flags::None)
ExecutorDeviceType device_type
void finalize(const CompilationOptions &co, Executor *executor)
std::string serialize_llvm_object(const T *llvm_obj)
llvm::Function * entry_point_func_
std::vector< llvm::Value * > generate_column_heads_load(const int num_columns, llvm::Value *byte_stream_arg, llvm::IRBuilder<> &ir_builder, llvm::LLVMContext &ctx)
llvm::Type * get_llvm_type_from_sql_column_type(const SQLTypeInfo elem_ti, llvm::LLVMContext &ctx)
bool g_enable_watchdog false
std::vector< Analyzer::Expr * > target_exprs
void compile(const TableFunctionExecutionUnit &exe_unit, const CompilationOptions &co, Executor *executor)