19 #include <llvm/IR/InstIterator.h>
20 #include <llvm/IR/Verifier.h>
21 #include <llvm/Support/raw_os_ostream.h>
22 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
24 #include <boost/algorithm/string.hpp>
33 const auto pi8_type = llvm::PointerType::get(
get_int_type(8, ctx), 0);
34 const auto ppi8_type = llvm::PointerType::get(pi8_type, 0);
35 const auto pi64_type = llvm::PointerType::get(
get_int_type(64, ctx), 0);
36 const auto ppi64_type = llvm::PointerType::get(pi64_type, 0);
39 const auto func_type = llvm::FunctionType::get(
41 {pi8_type, ppi8_type, pi64_type, ppi8_type, ppi64_type, ppi8_type, pi64_type},
44 auto func = llvm::Function::Create(func_type,
45 llvm::Function::ExternalLinkage,
46 "call_table_function",
48 auto arg_it = func->arg_begin();
49 const auto mgr_arg = &*arg_it;
50 mgr_arg->setName(
"mgr_ptr");
51 const auto input_cols_arg = &*(++arg_it);
52 input_cols_arg->setName(
"input_col_buffers");
53 const auto input_row_counts = &*(++arg_it);
54 input_row_counts->setName(
"input_row_counts");
55 const auto input_str_dict_proxies = &*(++arg_it);
56 input_str_dict_proxies->setName(
"input_str_dict_proxies");
57 const auto output_buffers = &*(++arg_it);
58 output_buffers->setName(
"output_buffers");
59 const auto output_str_dict_proxies = &*(++arg_it);
60 output_str_dict_proxies->setName(
"output_str_dict_proxies");
61 const auto output_row_count = &*(++arg_it);
62 output_row_count->setName(
"output_row_count");
67 llvm::LLVMContext& ctx) {
68 if (elem_ti.
is_fp()) {
85 LOG(
FATAL) <<
"get_llvm_type_from_sql_column_type: not implemented for "
92 llvm::Value* value_ptr,
93 llvm::IRBuilder<>& ir_builder) {
94 if (value_ptr !=
nullptr) {
95 if (value_ptr->getType() == member_llvm_type->getPointerElementType()) {
96 ir_builder.CreateStore(value_ptr, member_ptr);
98 auto tmp = ir_builder.CreateBitCast(value_ptr, member_llvm_type);
99 ir_builder.CreateStore(tmp, member_ptr);
102 ir_builder.CreateStore(llvm::Constant::getNullValue(member_llvm_type), member_ptr);
108 int64_t default_value,
109 llvm::LLVMContext& ctx,
110 llvm::IRBuilder<>& ir_builder) {
111 llvm::Value* val =
nullptr;
112 if (value !=
nullptr) {
113 auto value_type = value->getType();
114 if (value_type->isPointerTy()) {
115 CHECK(value_type->getPointerElementType()->isIntegerTy(64));
116 val = ir_builder.CreateLoad(value->getType()->getPointerElementType(), value);
118 CHECK(value_type->isIntegerTy(64));
121 ir_builder.CreateStore(val, member_ptr);
124 llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), default_value,
true);
125 ir_builder.CreateStore(const_default, member_ptr);
129 std::tuple<llvm::Value*, llvm::Value*>
alloc_column(std::string col_name,
132 llvm::Value* data_ptr,
133 llvm::Value* data_size,
134 llvm::Value* data_str_dict_proxy_ptr,
135 llvm::LLVMContext& ctx,
136 llvm::IRBuilder<>& ir_builder) {
150 const bool is_text_encoding_dict_type =
153 llvm::StructType* col_struct_type;
156 if (is_text_encoding_dict_type) {
157 col_struct_type = llvm::StructType::get(
161 llvm::Type::getInt64Ty(ctx),
162 llvm::Type::getInt8PtrTy(ctx)
166 llvm::StructType::get(ctx,
169 llvm::Type::getInt64Ty(ctx)
173 auto col = ir_builder.CreateAlloca(col_struct_type);
174 col->setName(col_name);
175 auto col_ptr_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 0);
176 auto col_sz_ptr = ir_builder.CreateStructGEP(col_struct_type, col, 1);
177 auto col_str_dict_ptr = is_text_encoding_dict_type
178 ? ir_builder.CreateStructGEP(col_struct_type, col, 2)
180 col_ptr_ptr->setName(col_name +
".ptr");
181 col_sz_ptr->setName(col_name +
".sz");
182 if (is_text_encoding_dict_type) {
183 col_str_dict_ptr->setName(col_name +
".string_dict_proxy");
188 if (is_text_encoding_dict_type) {
190 llvm::Type::getInt8PtrTy(ctx),
191 data_str_dict_proxy_ptr,
194 auto col_ptr = ir_builder.CreatePointerCast(
195 col_ptr_ptr, llvm::PointerType::get(llvm::Type::getInt8Ty(ctx), 0));
196 col_ptr->setName(col_name +
"_ptr");
197 return {col, col_ptr};
202 llvm::Value* data_ptrs,
204 llvm::Value* data_size,
205 llvm::Value* data_str_dict_proxy_ptrs,
206 llvm::LLVMContext& ctx,
207 llvm::IRBuilder<>& ir_builder) {
214 llvm::Type* data_ptrs_llvm_type = llvm::Type::getInt8PtrTy(ctx);
215 const bool is_text_encoding_dict_type =
219 llvm::StructType* col_list_struct_type =
220 is_text_encoding_dict_type
221 ? llvm::StructType::get(
225 llvm::Type::getInt64Ty(ctx),
226 llvm::Type::getInt64Ty(ctx),
229 : llvm::StructType::get(ctx,
232 llvm::Type::getInt64Ty(ctx),
233 llvm::Type::getInt64Ty(ctx)
236 auto col_list = ir_builder.CreateAlloca(col_list_struct_type);
237 col_list->setName(col_list_name);
238 auto col_list_ptr_ptr = ir_builder.CreateStructGEP(col_list_struct_type, col_list, 0);
239 auto col_list_length_ptr =
240 ir_builder.CreateStructGEP(col_list_struct_type, col_list, 1);
241 auto col_list_size_ptr = ir_builder.CreateStructGEP(col_list_struct_type, col_list, 2);
242 auto col_str_dict_ptr_ptr =
243 is_text_encoding_dict_type
244 ? ir_builder.CreateStructGEP(col_list_struct_type, col_list, 3)
247 col_list_ptr_ptr->setName(col_list_name +
".ptrs");
248 col_list_length_ptr->setName(col_list_name +
".length");
249 col_list_size_ptr->setName(col_list_name +
".size");
250 if (is_text_encoding_dict_type) {
251 col_str_dict_ptr_ptr->setName(col_list_name +
".string_dict_proxies");
257 auto const_length = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), length,
true);
258 ir_builder.CreateStore(const_length, col_list_length_ptr);
262 if (is_text_encoding_dict_type) {
264 data_str_dict_proxy_ptrs->getType(),
265 data_str_dict_proxy_ptrs,
269 auto col_list_ptr = ir_builder.CreatePointerCast(
270 col_list_ptr_ptr, llvm::PointerType::get(llvm::Type::getInt8Ty(ctx), 0));
271 col_list_ptr->setName(col_list_name +
"_ptrs");
275 std::string
exprsKey(
const std::vector<Analyzer::Expr*>& exprs) {
277 for (
const auto& expr : exprs) {
278 const auto& ti = expr->get_type_info();
279 result += ti.to_string() +
", ";
288 bool emit_only_preflight_fn) {
307 auto cgen_state =
executor_->getCgenStatePtr();
309 CHECK(cgen_state->module_ ==
nullptr);
310 cgen_state->set_module_shallow_copy(
executor_->get_rt_module());
317 CHECK(!emit_only_preflight_fn);
320 std::shared_ptr<CompilationContext> code;
322 code =
finalize(emit_only_preflight_fn);
323 }
catch (
const std::exception& e) {
337 auto mod =
executor_->get_rt_udf_module(is_gpu).get();
338 if (mod !=
nullptr) {
339 auto* flag = mod->getModuleFlag(
"pass_column_arguments_by_value");
340 if (
auto* cnt = llvm::mdconst::extract_or_null<llvm::ConstantInt>(flag)) {
341 return cnt->getZExtValue();
351 const std::vector<llvm::Value*>& func_args,
352 llvm::BasicBlock* bb_exit,
353 llvm::Value* output_row_count_ptr,
354 bool emit_only_preflight_fn) {
355 auto cgen_state =
executor_->getCgenStatePtr();
357 llvm::LLVMContext& ctx = cgen_state->context_;
358 llvm::IRBuilder<>* ir_builder = &cgen_state->ir_builder_;
360 std::string func_name =
363 llvm::Value* table_func_return =
364 cgen_state->emitExternalCall(func_name,
get_int_type(32, ctx), func_args);
366 table_func_return->setName(emit_only_preflight_fn ?
"preflight_check_func_ret"
372 llvm::BasicBlock* bb_exit_0 =
375 llvm::Constant* const_zero =
376 llvm::ConstantInt::get(table_func_return->getType(), 0,
true);
377 llvm::Value* is_ok = ir_builder->CreateICmpSGE(table_func_return, const_zero);
378 ir_builder->CreateCondBr(is_ok, bb_exit_0, bb_exit);
380 ir_builder->SetInsertPoint(bb_exit_0);
382 ir_builder->CreateIntCast(table_func_return,
get_int_type(64, ctx),
true);
383 ir_builder->CreateStore(r, output_row_count_ptr);
384 ir_builder->CreateRet(const_zero);
386 ir_builder->SetInsertPoint(bb_exit);
387 ir_builder->CreateRet(table_func_return);
392 bool emit_only_preflight_fn) {
397 const auto mgr_ptr = &*arg_it;
398 const auto input_cols_arg = &*(++arg_it);
399 const auto input_row_counts_arg = &*(++arg_it);
400 const auto input_str_dict_proxies_arg = &*(++arg_it);
401 const auto output_buffers_arg = &*(++arg_it);
402 const auto output_str_dict_proxies_arg = &*(++arg_it);
403 const auto output_row_count_ptr = &*(++arg_it);
404 auto cgen_state =
executor_->getCgenStatePtr();
406 auto& ctx = cgen_state->context_;
408 llvm::BasicBlock* bb_entry =
410 cgen_state->ir_builder_.SetInsertPoint(bb_entry);
412 llvm::BasicBlock* bb_exit = llvm::BasicBlock::Create(ctx,
".exit",
entry_point_func_);
414 llvm::BasicBlock* func_body_bb = llvm::BasicBlock::Create(
415 ctx,
".func_body0", cgen_state->ir_builder_.GetInsertBlock()->getParent());
417 cgen_state->ir_builder_.SetInsertPoint(bb_entry);
418 cgen_state->ir_builder_.CreateBr(func_body_bb);
420 cgen_state->ir_builder_.SetInsertPoint(func_body_bb);
422 exe_unit.
input_exprs.size(), input_cols_arg, cgen_state->ir_builder_, ctx);
425 exe_unit.
input_exprs.size(), input_row_counts_arg, cgen_state->ir_builder_, ctx);
427 auto input_str_dict_proxy_heads = std::vector<llvm::Value*>();
430 input_str_dict_proxies_arg,
431 cgen_state->ir_builder_,
437 std::vector<llvm::Value*> func_args;
438 size_t func_arg_index = 0;
440 func_args.push_back(mgr_ptr);
445 for (
size_t i = 0; i < exe_unit.
input_exprs.size(); i++) {
447 const auto& ti = expr->get_type_info();
448 if (col_index == -1) {
452 auto r = cgen_state->ir_builder_.CreateBitCast(
454 llvm::LoadInst* scalar_fp = cgen_state->ir_builder_.CreateLoad(
455 r->getType()->getPointerElementType(),
458 func_args.push_back(scalar_fp);
460 }
else if (ti.is_integer() || ti.is_boolean() || ti.is_timestamp() ||
461 ti.is_timeinterval()) {
462 auto r = cgen_state->ir_builder_.CreateBitCast(
464 llvm::LoadInst* scalar_int = cgen_state->ir_builder_.CreateLoad(
465 r->getType()->getPointerElementType(),
468 func_args.push_back(scalar_int);
470 }
else if (ti.is_text_encoding_none()) {
472 cgen_state->ir_builder_.CreateBitCast(col_heads[i],
get_int_ptr_type(64, ctx));
473 auto varchar_ptr = cgen_state->ir_builder_.CreateGEP(
474 col_heads[i]->getType()->getScalarType()->getPointerElementType(),
476 cgen_state->llInt(8));
477 auto [varchar_struct, varchar_struct_ptr] =
alloc_column(
478 std::string(
"input_varchar_literal.") +
std::to_string(func_arg_index),
485 cgen_state->ir_builder_);
487 (pass_column_by_value
488 ? cgen_state->ir_builder_.CreateLoad(
489 varchar_struct->getType()->getPointerElementType(), varchar_struct)
490 : varchar_struct_ptr));
492 }
else if (ti.is_column()) {
500 : input_str_dict_proxy_heads[i],
502 cgen_state->ir_builder_);
503 func_args.push_back((pass_column_by_value
504 ? cgen_state->ir_builder_.CreateLoad(
505 col->getType()->getPointerElementType(), col)
508 }
else if (ti.is_column_list()) {
509 if (col_index == -1) {
516 input_str_dict_proxy_heads[i],
518 cgen_state->ir_builder_);
519 func_args.push_back(col_list);
522 if (col_index + 1 == ti.get_dimension()) {
526 throw std::runtime_error(
527 "Only integer and floating point columns or scalars are supported as inputs to "
533 auto output_str_dict_proxy_heads =
536 output_str_dict_proxies_arg,
537 cgen_state->ir_builder_,
539 : std::vector<llvm::Value*>();
541 std::vector<llvm::Value*> output_col_args;
542 for (
size_t i = 0; i < exe_unit.
target_exprs.size(); i++) {
543 auto* gep = cgen_state->ir_builder_.CreateGEP(
544 output_buffers_arg->getType()->getScalarType()->getPointerElementType(),
546 cgen_state->llInt(i));
548 cgen_state->ir_builder_.CreateLoad(gep->getType()->getPointerElementType(), gep);
550 const auto& ti = expr->get_type_info();
551 CHECK(!ti.is_column());
552 CHECK(!ti.is_column_list());
555 CHECK_EQ(ti.supportsFlatBuffer(), ti.usesFlatBuffer()) << ti;
564 output_row_count_ptr,
568 cgen_state->ir_builder_);
570 cgen_state->emitExternalCall(
571 "TableFunctionManager_register_output_column",
572 llvm::Type::getVoidTy(ctx),
573 {mgr_ptr, llvm::ConstantInt::get(
get_int_type(32, ctx), i,
true), col_ptr});
575 output_col_args.push_back((pass_column_by_value ? col : col_ptr));
583 cgen_state->emitExternalCall(
584 "TableFunctionManager_set_output_row_size",
585 llvm::Type::getVoidTy(ctx),
587 cgen_state->ir_builder_.CreateLoad(
588 output_row_count_ptr->getType()->getPointerElementType(),
589 output_row_count_ptr)});
592 if (!emit_only_preflight_fn) {
593 for (
auto& col : output_col_args) {
594 func_args.push_back((pass_column_by_value
595 ? cgen_state->ir_builder_.CreateLoad(
596 col->getType()->getPointerElementType(), col)
602 exe_unit, func_args, bb_exit, output_row_count_ptr, emit_only_preflight_fn);
614 std::vector<llvm::Type*> arg_types;
618 [&arg_types](
const auto& arg) { arg_types.push_back(arg.getType()); });
621 auto cgen_state =
executor_->getCgenStatePtr();
623 auto& ctx = cgen_state->context_;
625 std::vector<llvm::Type*> wrapper_arg_types(arg_types.size() + 1);
626 wrapper_arg_types[0] = llvm::PointerType::get(
get_int_type(32, ctx), 0);
627 wrapper_arg_types[1] = arg_types[0];
629 for (
size_t i = 1; i < arg_types.size(); ++i) {
630 wrapper_arg_types[i + 1] = arg_types[i];
634 llvm::FunctionType::get(llvm::Type::getVoidTy(ctx), wrapper_arg_types,
false);
636 llvm::Function::ExternalLinkage,
638 cgen_state->module_);
640 auto wrapper_bb_entry = llvm::BasicBlock::Create(ctx,
".entry",
kernel_func_, 0);
641 llvm::IRBuilder<> b(ctx);
642 b.SetInsertPoint(wrapper_bb_entry);
643 std::vector<llvm::Value*> loaded_args = {
kernel_func_->arg_begin() + 1};
644 for (
size_t i = 2; i < wrapper_arg_types.size(); ++i) {
653 bool emit_only_preflight_fn) {
659 auto cgen_state =
executor_->getCgenStatePtr();
661 if (
executor_->has_rt_udf_module(is_gpu)) {
663 *(cgen_state->module_),
665 llvm::Linker::Flags::OverrideFromSrc);
668 LOG(
IR) << (emit_only_preflight_fn ?
"Pre Flight Function Entry Point IR\n"
669 :
"Table Function Entry Point IR\n")
671 std::shared_ptr<CompilationContext> code;
690 auto cpu_code = std::make_shared<CpuCompilationContext>(std::move(ee));
694 LOG(
IR) <<
"End of IR";
llvm::Type * get_fp_ptr_type(const int width, llvm::LLVMContext &context)
llvm::Function * kernel_func_
std::string exprsKey(const std::vector< Analyzer::Expr * > &exprs)
HOST DEVICE int get_size() const
bool is_timestamp() const
void generateEntryPoint(const TableFunctionExecutionUnit &exe_unit, bool emit_only_preflight_fn)
void initialize_ptr_member(llvm::Value *member_ptr, llvm::Type *member_llvm_type, llvm::Value *value_ptr, llvm::IRBuilder<> &ir_builder)
bool passColumnsByValue(const TableFunctionExecutionUnit &exe_unit)
std::vector< Analyzer::Expr * > input_exprs
const table_functions::TableFunction table_func
std::tuple< llvm::Value *, llvm::Value * > alloc_column(std::string col_name, const size_t index, const SQLTypeInfo &data_target_info, llvm::Value *data_ptr, llvm::Value *data_size, llvm::Value *data_str_dict_proxy_ptr, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder)
llvm::Function * generate_entry_point(const CgenState *cgen_state)
std::shared_ptr< CompilationContext > finalize(bool emit_only_preflight_fn)
std::vector< std::string > CodeCacheKey
std::string toString(const QueryDescriptionType &type)
static ExecutionEngineWrapper generateNativeCPUCode(llvm::Function *func, const std::unordered_set< llvm::Function * > &live_funcs, const CompilationOptions &co)
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
llvm::Value * alloc_column_list(std::string col_list_name, const SQLTypeInfo &data_target_info, llvm::Value *data_ptrs, int length, llvm::Value *data_size, llvm::Value *data_str_dict_proxy_ptrs, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder)
bool hasOutputSizeKnownPreLaunch() const
void verify_function_ir(const llvm::Function *func)
size_t get_bit_width(const SQLTypeInfo &ti)
llvm::LLVMContext & context_
const CompilationOptions & co_
std::string getPreFlightFnName() const
bool usesFlatBuffer() const
static void link_udf_module(const std::unique_ptr< llvm::Module > &udf_module, llvm::Module &module, CgenState *cgen_state, llvm::Linker::Flags flags=llvm::Linker::Flags::None)
void generateTableFunctionCall(const TableFunctionExecutionUnit &exe_unit, const std::vector< llvm::Value * > &func_args, llvm::BasicBlock *bb_exit, llvm::Value *output_row_count_ptr, bool emit_only_preflight_fn)
std::string getName(const bool drop_suffix=false, const bool lower=false) const
ExecutorDeviceType device_type
HOST DEVICE EncodingType get_compression() const
std::string serialize_llvm_object(const T *llvm_obj)
static std::shared_ptr< GpuCompilationContext > generateNativeGPUCode(Executor *executor, llvm::Function *func, llvm::Function *wrapper_func, const std::unordered_set< llvm::Function * > &live_funcs, const bool is_gpu_smem_used, const CompilationOptions &co, const GPUTarget &gpu_target)
bool hasPreFlightOutputSizer() const
llvm::Function * entry_point_func_
std::vector< llvm::Value * > generate_column_heads_load(const int num_columns, llvm::Value *byte_stream_arg, llvm::IRBuilder<> &ir_builder, llvm::LLVMContext &ctx)
llvm::Type * get_llvm_type_from_sql_column_type(const SQLTypeInfo elem_ti, llvm::LLVMContext &ctx)
#define DEBUG_TIMER(name)
void initialize_int64_member(llvm::Value *member_ptr, llvm::Value *value, int64_t default_value, llvm::LLVMContext &ctx, llvm::IRBuilder<> &ir_builder)
static std::shared_ptr< QueryEngine > getInstance()
std::shared_ptr< CompilationContext > compile(const TableFunctionExecutionUnit &exe_unit, bool emit_only_preflight_fn)
std::vector< Analyzer::Expr * > target_exprs
bool is_text_encoding_none() const
llvm::Type * get_int_ptr_type(const int width, llvm::LLVMContext &context)