#include <CgenState.h>

Collaboration diagram for CgenState:

Classes
struct	FunctionOperValue

struct	HoistedLiteralLoadLocator

Public Types
using	LiteralValue = boost::variant< int8_t, int16_t, int32_t, int64_t, float, double, std::pair< std::string, int >, std::string, std::vector< double >, std::vector< int32_t >, std::vector< int8_t >, std::pair< std::vector< int8_t >, int >>

using	LiteralValues = std::vector< LiteralValue >

Public Member Functions
	CgenState (const size_t num_query_infos, const bool contains_left_deep_outer_join, Executor *executor)

	CgenState (const size_t num_query_infos, const bool contains_left_deep_outer_join)

	CgenState (llvm::LLVMContext &context)

std::tuple< size_t, size_t >	getOrAddLiteral (const Analyzer::Constant *constant, const EncodingType enc_type, const int dict_id, const int device_id)

const std::unordered_map< int, LiteralValues > &	getLiterals () const

llvm::Value *	addStringConstant (const std::string &str)

const StringDictionaryTranslationMgr *	moveStringDictionaryTranslationMgr (std::unique_ptr< const StringDictionaryTranslationMgr > &&str_dict_translation_mgr)

const InValuesBitmap *	addInValuesBitmap (std::unique_ptr< InValuesBitmap > &in_values_bitmap)

void	moveInValuesBitmap (std::unique_ptr< const InValuesBitmap > &in_values_bitmap)

llvm::Value *	emitExternalCall (const std::string &fname, llvm::Type ret_type, const std::vector< llvm::Value > args, const std::vector< llvm::Attribute::AttrKind > &fnattrs={}, const bool has_struct_return=false)

llvm::Value *	emitCall (const std::string &fname, const std::vector< llvm::Value * > &args)

llvm::Value *	emitEntryCall (const std::string &fname, const std::vector< llvm::Value * > &args)

size_t	getLiteralBufferUsage (const int device_id)

llvm::Value *	castToTypeIn (llvm::Value *val, const size_t bit_width)

std::pair< llvm::ConstantInt , llvm::ConstantInt >	inlineIntMaxMin (const size_t byte_width, const bool is_signed)

llvm::ConstantInt *	inlineIntNull (const SQLTypeInfo &)

llvm::ConstantFP *	inlineFpNull (const SQLTypeInfo &)

llvm::Constant *	inlineNull (const SQLTypeInfo &)

template<class T >
llvm::ConstantInt *	llInt (const T v) const

llvm::ConstantFP *	llFp (const float v) const

llvm::ConstantFP *	llFp (const double v) const

llvm::ConstantInt *	llBool (const bool v) const

void	emitErrorCheck (llvm::Value condition, llvm::Value errorCode, std::string label)

std::vector< std::string >	gpuFunctionsToReplace (llvm::Function *fn)

void	replaceFunctionForGpu (const std::string &fcn_to_replace, llvm::Function *fn)

std::shared_ptr< Executor >	getExecutor () const

llvm::LLVMContext &	getExecutorContext () const

void	set_module_shallow_copy (const std::unique_ptr< llvm::Module > &module, bool always_clone=false)

void	maybeCloneFunctionRecursive (llvm::Function *fn)

Static Public Member Functions
static size_t	literalBytes (const CgenState::LiteralValue &lit)

static size_t	addAligned (const size_t off_in, const size_t alignment)

Public Attributes
size_t	executor_id_

llvm::Module *	module_

llvm::Function *	row_func_

llvm::Function *	filter_func_

llvm::Function *	current_func_

llvm::BasicBlock *	row_func_bb_

llvm::BasicBlock *	filter_func_bb_

llvm::CallInst *	row_func_call_

llvm::CallInst *	filter_func_call_

std::vector< llvm::Function * >	helper_functions_

llvm::LLVMContext &	context_

llvm::ValueToValueMapTy	vmap_

llvm::IRBuilder	ir_builder_

std::unordered_map< size_t, std::vector< llvm::Value * > >	fetch_cache_

std::vector< FunctionOperValue >	ext_call_cache_

std::vector< llvm::Value * >	group_by_expr_cache_

std::vector< llvm::Value * >	str_constants_

std::vector< llvm::Value * >	frag_offsets_

const bool	contains_left_deep_outer_join_

std::vector< llvm::Value * >	outer_join_match_found_per_level_

std::unordered_map< int, llvm::Value * >	scan_idx_to_hash_pos_

InsertionOrderedMap	filter_func_args_

std::vector< std::unique_ptr < const InValuesBitmap > >	in_values_bitmaps_

std::vector< std::unique_ptr < const StringDictionaryTranslationMgr > >	str_dict_translation_mgrs_

std::map< std::pair < llvm::Value , llvm::Value > , ArrayLoadCodegen >	array_load_cache_

std::unordered_map < std::string, llvm::Value * >	geo_target_cache_

bool	needs_error_check_

bool	needs_geos_

llvm::Function *	query_func_

llvm::IRBuilder	query_func_entry_ir_builder_

std::unordered_map< int, std::vector< llvm::Value * > >	query_func_literal_loads_

std::unordered_map < llvm::Value *, HoistedLiteralLoadLocator >	row_func_hoisted_literals_

Private Member Functions
template<class T >
std::tuple< size_t, size_t >	getOrAddLiteral (const T &val, const int device_id)

Private Attributes
std::unordered_map< int, LiteralValues >	literals_

std::unordered_map< int, size_t >	literal_bytes_

Detailed Description

Definition at line 38 of file CgenState.h.

Member Typedef Documentation

using CgenState::LiteralValue = boost::variant<int8_t, int16_t, int32_t, int64_t, float, double, std::pair<std::string, int>, std::string, std::vector<double>, std::vector<int32_t>, std::vector<int8_t>, std::pair<std::vector<int8_t>, int>>

Definition at line 182 of file CgenState.h.

using CgenState::LiteralValues = std::vector<LiteralValue>

Definition at line 183 of file CgenState.h.

Constructor & Destructor Documentation

CgenState::CgenState	(	const size_t	num_query_infos,
		const bool	contains_left_deep_outer_join,
		Executor *	executor
	)

Definition at line 25 of file CgenState.cpp.

     : executor_id_(executor->getExecutorId())
     , module_(nullptr)
     , row_func_(nullptr)
     , filter_func_(nullptr)
     , current_func_(nullptr)
     , row_func_bb_(nullptr)
     , filter_func_bb_(nullptr)
     , row_func_call_(nullptr)
     , filter_func_call_(nullptr)
     , context_(executor->getContext())
     , ir_builder_(context_)
     , contains_left_deep_outer_join_(contains_left_deep_outer_join)
     , outer_join_match_found_per_level_(std::max(num_query_infos, size_t(1)) - 1)
     , needs_error_check_(false)
     , needs_geos_(false)
     , query_func_(nullptr)
     , query_func_entry_ir_builder_(context_){};

CgenState::CgenState	(	const size_t	num_query_infos,
		const bool	contains_left_deep_outer_join
	)

Definition at line 46 of file CgenState.cpp.

     : CgenState(num_query_infos,
                 contains_left_deep_outer_join,
                 Executor::getExecutor(Executor::UNITARY_EXECUTOR_ID).get()) {}

CgenState::CgenState ( llvm::LLVMContext & context )

Definition at line 52 of file CgenState.cpp.

     : executor_id_(Executor::INVALID_EXECUTOR_ID)
     , module_(nullptr)
     , row_func_(nullptr)
     , context_(context)
     , ir_builder_(context_)
     , contains_left_deep_outer_join_(false)
     , needs_error_check_(false)
     , needs_geos_(false)
     , query_func_(nullptr)
     , query_func_entry_ir_builder_(context_){};

Member Function Documentation

static size_t CgenState::addAligned	(	const size_t	off_in,
		const size_t	alignment
	)

inlinestatic

Definition at line 505 of file CgenState.h.

Referenced by getOrAddLiteral(), and Executor::serializeLiterals().

                                                                         {
     size_t off = off_in;
     if (off % alignment != 0) {
       off += (alignment - off % alignment);
     }
     return off + alignment;
   }

Here is the caller graph for this function:

const InValuesBitmap* CgenState::addInValuesBitmap ( std::unique_ptr< InValuesBitmap > & in_values_bitmap )

inline

Definition at line 202 of file CgenState.h.

References in_values_bitmaps_.

Referenced by CodeGenerator::codegen().

                                                        {
     if (in_values_bitmap->isEmpty()) {
       return in_values_bitmap.get();
     }
     in_values_bitmaps_.emplace_back(std::move(in_values_bitmap));
     return in_values_bitmaps_.back().get();
   }

Here is the caller graph for this function:

llvm::Value* CgenState::addStringConstant ( const std::string & str )

inline

Definition at line 187 of file CgenState.h.

References context_, get_int_type(), ir_builder_, str_constants_, and to_string().

Referenced by CodeGenerator::codegen().

                                                      {
     llvm::Value* str_lv = ir_builder_.CreateGlobalString(
         str, "str_const_" + std::to_string(std::hash<std::string>()(str)));
     auto i8_ptr = llvm::PointerType::get(get_int_type(8, context_), 0);
     str_constants_.push_back(str_lv);
     str_lv = ir_builder_.CreateBitCast(str_lv, i8_ptr);
     return str_lv;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::Value * CgenState::castToTypeIn	(	llvm::Value *	val,
		const size_t	bit_width
	)

Definition at line 152 of file CgenState.cpp.

References CHECK, context_, get_int_type(), and ir_builder_.

Referenced by CodeGenerator::codegenCast(), CodeGenerator::codegenLogical(), CodeGenerator::codegenLogicalShortCircuit(), and anonymous_namespace{WindowFunctionIR.cpp}::get_null_value_by_size_with_encoding().

                                                                         {
   auto src_bits = val->getType()->getScalarSizeInBits();
   if (src_bits == dst_bits) {
     return val;
   }
   if (val->getType()->isIntegerTy()) {
     return ir_builder_.CreateIntCast(
         val, get_int_type(dst_bits, context_), src_bits != 1);
   }
   // real (not dictionary-encoded) strings; store the pointer to the payload
   if (val->getType()->isPointerTy()) {
     return ir_builder_.CreatePointerCast(val, get_int_type(dst_bits, context_));
   }
 
   CHECK(val->getType()->isFloatTy() || val->getType()->isDoubleTy());
 
   llvm::Type* dst_type = nullptr;
   switch (dst_bits) {
     case 64:
       dst_type = llvm::Type::getDoubleTy(context_);
       break;
     case 32:
       dst_type = llvm::Type::getFloatTy(context_);
       break;
     default:
       CHECK(false);
   }
 
   return ir_builder_.CreateFPCast(val, dst_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::Value * CgenState::emitCall	(	const std::string &	fname,
		const std::vector< llvm::Value * > &	args
	)

Definition at line 219 of file CgenState.cpp.

References CHECK, ir_builder_, maybeCloneFunctionRecursive(), and module_.

                                                                     {
   // Get the function reference from the query module.
   auto func = module_->getFunction(fname);
   CHECK(func);
   // If the function called isn't external, clone the implementation from the runtime
   // module.
   maybeCloneFunctionRecursive(func);
 
   return ir_builder_.CreateCall(func, args);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::Value * CgenState::emitEntryCall	(	const std::string &	fname,
		const std::vector< llvm::Value * > &	args
	)

Definition at line 231 of file CgenState.cpp.

References CHECK, maybeCloneFunctionRecursive(), module_, and query_func_entry_ir_builder_.

Referenced by CodeGenerator::codegenHoistedConstantsLoads().

                                                                          {
   // Get the function reference from the query module.
   auto func = module_->getFunction(fname);
   CHECK(func);
   // If the function called isn't external, clone the implementation from the runtime
   // module.
   maybeCloneFunctionRecursive(func);
 
   return query_func_entry_ir_builder_.CreateCall(func, args);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void CgenState::emitErrorCheck	(	llvm::Value *	condition,
		llvm::Value *	errorCode,
		std::string	label
	)

Definition at line 243 of file CgenState.cpp.

References context_, current_func_, ir_builder_, and needs_error_check_.

                                                 {
   needs_error_check_ = true;
   auto check_ok = llvm::BasicBlock::Create(context_, label + "_ok", current_func_);
   auto check_fail = llvm::BasicBlock::Create(context_, label + "_fail", current_func_);
   ir_builder_.CreateCondBr(condition, check_ok, check_fail);
   ir_builder_.SetInsertPoint(check_fail);
   ir_builder_.CreateRet(errorCode);
   ir_builder_.SetInsertPoint(check_ok);
 }

llvm::Value* CgenState::emitExternalCall	(	const std::string &	fname,
		llvm::Type *	ret_type,
		const std::vector< llvm::Value * >	args,
		const std::vector< llvm::Attribute::AttrKind > &	fnattrs = `{}`,
		const bool	has_struct_return = `false`
	)

inline

Definition at line 217 of file CgenState.h.

                                                         {},
       const bool has_struct_return = false) {
     std::vector<llvm::Type*> arg_types;
     for (const auto arg : args) {
       CHECK(arg);
       arg_types.push_back(arg->getType());
     }
     auto func_ty = llvm::FunctionType::get(ret_type, arg_types, false);
     llvm::AttributeList attrs;
     if (!fnattrs.empty()) {
       std::vector<std::pair<unsigned, llvm::Attribute>> indexedAttrs;
       indexedAttrs.reserve(fnattrs.size());
       for (auto attr : fnattrs) {
         indexedAttrs.emplace_back(llvm::AttributeList::FunctionIndex,
                                   llvm::Attribute::get(context_, attr));
       }
       attrs = llvm::AttributeList::get(context_,
                                        {&indexedAttrs.front(), indexedAttrs.size()});
     }
 
     auto func_p = module_->getOrInsertFunction(fname, func_ty, attrs);
     CHECK(func_p);
     auto callee = func_p.getCallee();
     llvm::Function* func{nullptr};
     if (auto callee_cast = llvm::dyn_cast<llvm::ConstantExpr>(callee)) {
       // Get or insert function automatically adds a ConstantExpr cast if the return type
       // of the existing function does not match the supplied return type.
       CHECK(callee_cast->isCast());
       CHECK_EQ(callee_cast->getNumOperands(), size_t(1));
       func = llvm::dyn_cast<llvm::Function>(callee_cast->getOperand(0));
     } else {
       func = llvm::dyn_cast<llvm::Function>(callee);
     }
     CHECK(func);
     llvm::FunctionType* func_type = func_p.getFunctionType();
     CHECK(func_type);
     if (has_struct_return) {
       const auto arg_ti = func_type->getParamType(0);
       CHECK(arg_ti->isPointerTy() && arg_ti->getPointerElementType()->isStructTy());
       auto attr_list = func->getAttributes();
 #if 14 <= LLVM_VERSION_MAJOR
       llvm::AttrBuilder arr_arg_builder(context_, attr_list.getParamAttrs(0));
 #else
       llvm::AttrBuilder arr_arg_builder(attr_list.getParamAttributes(0));
 #endif
       arr_arg_builder.addAttribute(llvm::Attribute::StructRet);
       func->addParamAttrs(0, arr_arg_builder);
     }
     const size_t arg_start = has_struct_return ? 1 : 0;
     for (size_t i = arg_start; i < func->arg_size(); i++) {
       const auto arg_ti = func_type->getParamType(i);
       if (arg_ti->isPointerTy() && arg_ti->getPointerElementType()->isStructTy()) {
         auto attr_list = func->getAttributes();
 #if 14 <= LLVM_VERSION_MAJOR
         llvm::AttrBuilder arr_arg_builder(context_, attr_list.getParamAttrs(i));
 #else
         llvm::AttrBuilder arr_arg_builder(attr_list.getParamAttributes(i));
 #endif
         arr_arg_builder.addByValAttr(arg_ti->getPointerElementType());
         func->addParamAttrs(i, arr_arg_builder);
       }
     }
     llvm::Value* result = ir_builder_.CreateCall(func_p, args);
     // check the assumed type
     CHECK_EQ(result->getType(), ret_type);
     return result;
   }

Here is the caller graph for this function:

std::shared_ptr< Executor > CgenState::getExecutor ( ) const

Definition at line 375 of file CgenState.cpp.

References CHECK, executor_id_, Executor::getExecutor(), and Executor::INVALID_EXECUTOR_ID.

Referenced by getExecutorContext(), and maybeCloneFunctionRecursive().

                                                      {
   CHECK(executor_id_ != Executor::INVALID_EXECUTOR_ID);
   return Executor::getExecutor(executor_id_);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::LLVMContext & CgenState::getExecutorContext ( ) const

Definition at line 380 of file CgenState.cpp.

References getExecutor().

                                                      {
   return getExecutor()->getContext();
 }

Here is the call graph for this function:

size_t CgenState::getLiteralBufferUsage ( const int device_id )

inline

Definition at line 293 of file CgenState.h.

References literal_bytes_.

293 { return literal_bytes_[device_id]; }

CgenState::literal_bytes_

std::unordered_map< int, size_t > literal_bytes_

Definition: CgenState.h:535

const std::unordered_map<int, LiteralValues>& CgenState::getLiterals ( ) const

inline

Definition at line 185 of file CgenState.h.

References literals_.

185 { return literals_; }

CgenState::literals_

std::unordered_map< int, LiteralValues > literals_

Definition: CgenState.h:534

std::tuple<size_t, size_t> CgenState::getOrAddLiteral	(	const Analyzer::Constant *	constant,
		const EncodingType	enc_type,
		const int	dict_id,
		const int	device_id
	)

inline

Definition at line 46 of file CgenState.h.

References Datum::bigintval, Datum::boolval, CHECK, CHECK_EQ, decimal_to_int_type(), Datum::doubleval, Datum::floatval, Analyzer::Constant::get_constval(), Analyzer::Constant::get_is_null(), Analyzer::Expr::get_type_info(), Analyzer::Constant::get_value_list(), inline_fp_null_val(), inline_int_null_val(), Datum::intval, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kENCODING_DICT, kENCODING_GEOINT, kENCODING_NONE, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, Datum::smallintval, Datum::stringval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by CodeGenerator::codegenHoistedConstants().

                                                                   {
     const auto& ti = constant->get_type_info();
     const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
     switch (type) {
       case kBOOLEAN:
         return getOrAddLiteral(constant->get_is_null()
                                    ? int8_t(inline_int_null_val(ti))
                                    : int8_t(constant->get_constval().boolval ? 1 : 0),
                                device_id);
       case kTINYINT:
         return getOrAddLiteral(constant->get_is_null()
                                    ? int8_t(inline_int_null_val(ti))
                                    : constant->get_constval().tinyintval,
                                device_id);
       case kSMALLINT:
         return getOrAddLiteral(constant->get_is_null()
                                    ? int16_t(inline_int_null_val(ti))
                                    : constant->get_constval().smallintval,
                                device_id);
       case kINT:
         return getOrAddLiteral(constant->get_is_null() ? int32_t(inline_int_null_val(ti))
                                                        : constant->get_constval().intval,
                                device_id);
       case kBIGINT:
         return getOrAddLiteral(constant->get_is_null()
                                    ? int64_t(inline_int_null_val(ti))
                                    : constant->get_constval().bigintval,
                                device_id);
       case kFLOAT:
         return getOrAddLiteral(constant->get_is_null()
                                    ? float(inline_fp_null_val(ti))
                                    : constant->get_constval().floatval,
                                device_id);
       case kDOUBLE:
         return getOrAddLiteral(constant->get_is_null()
                                    ? inline_fp_null_val(ti)
                                    : constant->get_constval().doubleval,
                                device_id);
       case kCHAR:
       case kTEXT:
       case kVARCHAR:
         if (enc_type == kENCODING_DICT) {
           if (constant->get_is_null()) {
             return getOrAddLiteral(int32_t(inline_int_null_val(ti)), device_id);
           }
           return getOrAddLiteral(
               std::make_pair(*constant->get_constval().stringval, dict_id), device_id);
         }
         CHECK_EQ(kENCODING_NONE, enc_type);
         if (constant->get_is_null()) {
           throw std::runtime_error(
               "CHAR / VARCHAR NULL literal not supported in this context");  // TODO(alex):
                                                                              // support
                                                                              // null
         }
         return getOrAddLiteral(*constant->get_constval().stringval, device_id);
       case kTIME:
       case kTIMESTAMP:
       case kDATE:
       case kINTERVAL_DAY_TIME:
       case kINTERVAL_YEAR_MONTH:
         // TODO(alex): support null
         return getOrAddLiteral(constant->get_constval().bigintval, device_id);
       case kARRAY: {
         if (enc_type == kENCODING_NONE) {
           if (ti.get_subtype() == kDOUBLE) {
             std::vector<double> double_array_literal;
             for (const auto& value : constant->get_value_list()) {
               const auto c = dynamic_cast<const Analyzer::Constant*>(value.get());
               CHECK(c);
               double d = c->get_constval().doubleval;
               double_array_literal.push_back(d);
             }
             return getOrAddLiteral(double_array_literal, device_id);
           }
           if (ti.get_subtype() == kINT) {
             std::vector<int32_t> int32_array_literal;
             for (const auto& value : constant->get_value_list()) {
               const auto c = dynamic_cast<const Analyzer::Constant*>(value.get());
               CHECK(c);
               int32_t i = c->get_constval().intval;
               int32_array_literal.push_back(i);
             }
             return getOrAddLiteral(int32_array_literal, device_id);
           }
           if (ti.get_subtype() == kTINYINT) {
             std::vector<int8_t> int8_array_literal;
             for (const auto& value : constant->get_value_list()) {
               const auto c = dynamic_cast<const Analyzer::Constant*>(value.get());
               CHECK(c);
               int8_t i = c->get_constval().tinyintval;
               int8_array_literal.push_back(i);
             }
             if (ti.get_comp_param() == 64) {
               return getOrAddLiteral(std::make_pair(int8_array_literal, 64), device_id);
             }
             return getOrAddLiteral(int8_array_literal, device_id);
           }
           throw std::runtime_error("Unsupported literal array");
         }
         if (enc_type == kENCODING_GEOINT) {
           if (ti.get_subtype() == kTINYINT) {
             std::vector<int8_t> int8_array_literal;
             for (const auto& value : constant->get_value_list()) {
               const auto c = dynamic_cast<const Analyzer::Constant*>(value.get());
               CHECK(c);
               int8_t i = c->get_constval().tinyintval;
               int8_array_literal.push_back(i);
             }
             if (ti.get_comp_param() == 32) {
               return getOrAddLiteral(std::make_pair(int8_array_literal, 32), device_id);
             }
             return getOrAddLiteral(int8_array_literal, device_id);
           }
         }
         throw std::runtime_error("Encoded literal arrays are not supported");
       }
       default:
         abort();
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

template<class T >

std::tuple<size_t, size_t> CgenState::getOrAddLiteral	(	const T &	val,
		const int	device_id
	)

inlineprivate

Definition at line 517 of file CgenState.h.

References addAligned(), literal_bytes_, literalBytes(), and literals_.

                                                                               {
     const LiteralValue var_val(val);
     size_t literal_found_off{0};
     auto& literals = literals_[device_id];
     for (const auto& literal : literals) {
       const auto lit_bytes = literalBytes(literal);
       literal_found_off = addAligned(literal_found_off, lit_bytes);
       if (literal == var_val) {
         return {literal_found_off - lit_bytes, lit_bytes};
       }
     }
     literals.emplace_back(val);
     const auto lit_bytes = literalBytes(var_val);
     literal_bytes_[device_id] = addAligned(literal_bytes_[device_id], lit_bytes);
     return {literal_bytes_[device_id] - lit_bytes, lit_bytes};
   }

Here is the call graph for this function:

std::vector< std::string > CgenState::gpuFunctionsToReplace ( llvm::Function * fn )

Definition at line 308 of file CgenState.cpp.

References CHECK, and anonymous_namespace{CgenState.cpp}::gpu_replacement_functions.

Referenced by spatial_type::Transform::codegen().

                                                                       {
   std::vector<std::string> ret;
 
   CHECK(fn);
   CHECK(!fn->isDeclaration());
 
   for (auto& basic_block : *fn) {
     auto& inst_list = basic_block.getInstList();
     for (auto inst_itr = inst_list.begin(); inst_itr != inst_list.end(); ++inst_itr) {
       if (auto call_inst = llvm::dyn_cast<llvm::CallInst>(inst_itr)) {
         auto called_fcn = call_inst->getCalledFunction();
         CHECK(called_fcn);
 
         if (gpu_replacement_functions.find(called_fcn->getName().str()) !=
             gpu_replacement_functions.end()) {
           ret.emplace_back(called_fcn->getName());
         }
       }
     }
   }
   return ret;
 }

Here is the caller graph for this function:

llvm::ConstantFP * CgenState::inlineFpNull ( const SQLTypeInfo & type_info )

Definition at line 106 of file CgenState.cpp.

References CHECK, SQLTypeInfo::get_type(), SQLTypeInfo::is_fp(), kDOUBLE, kFLOAT, llFp(), NULL_DOUBLE, and NULL_FLOAT.

Referenced by CodeGenerator::codegenArrayAt(), CodeGenerator::codegenCastFromFp(), CodeGenerator::codegenCastToFp(), CodeGenerator::codegenFixedLengthColVarInWindow(), CodeGenerator::codegenUMinus(), CodeGenerator::codegenWidthBucketExpr(), CodeGenerator::endArgsNullcheck(), inlineNull(), and CodeGenerator::NullCheckCodegen::NullCheckCodegen().

                                                                     {
   CHECK(type_info.is_fp());
   switch (type_info.get_type()) {
     case kFLOAT:
       return llFp(NULL_FLOAT);
     case kDOUBLE:
       return llFp(NULL_DOUBLE);
     default:
       abort();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair< llvm::ConstantInt , llvm::ConstantInt > CgenState::inlineIntMaxMin	(	const size_t	byte_width,
		const bool	is_signed
	)

Definition at line 123 of file CgenState.cpp.

References CHECK_EQ, context_, inline_int_max_min(), inline_uint_max_min(), and ll_int().

Referenced by CodeGenerator::codegenAdd(), CodeGenerator::codegenCastBetweenIntTypesOverflowChecks(), CodeGenerator::codegenDiv(), CodeGenerator::codegenMul(), CodeGenerator::codegenSub(), and CodeGenerator::codegenUMinus().

                           {
   int64_t max_int{0}, min_int{0};
   if (is_signed) {
     std::tie(max_int, min_int) = inline_int_max_min(byte_width);
   } else {
     uint64_t max_uint{0}, min_uint{0};
     std::tie(max_uint, min_uint) = inline_uint_max_min(byte_width);
     max_int = static_cast<int64_t>(max_uint);
     CHECK_EQ(uint64_t(0), min_uint);
   }
   switch (byte_width) {
     case 1:
       return std::make_pair(::ll_int(static_cast<int8_t>(max_int), context_),
                             ::ll_int(static_cast<int8_t>(min_int), context_));
     case 2:
       return std::make_pair(::ll_int(static_cast<int16_t>(max_int), context_),
                             ::ll_int(static_cast<int16_t>(min_int), context_));
     case 4:
       return std::make_pair(::ll_int(static_cast<int32_t>(max_int), context_),
                             ::ll_int(static_cast<int32_t>(min_int), context_));
     case 8:
       return std::make_pair(::ll_int(max_int, context_), ::ll_int(min_int, context_));
     default:
       abort();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::ConstantInt * CgenState::inlineIntNull ( const SQLTypeInfo & type_info )

Definition at line 64 of file CgenState.cpp.

References CHECK, SQLTypeInfo::get_compression(), SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), inline_int_null_val(), SQLTypeInfo::is_string(), kARRAY, kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kENCODING_DICT, kENCODING_FIXED, kENCODING_NONE, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, llInt(), and run_benchmark_import::type.

                                                                       {
   auto type = type_info.get_type();
   if (type_info.is_string()) {
     switch (type_info.get_compression()) {
       case kENCODING_DICT:
         return llInt(static_cast<int32_t>(inline_int_null_val(type_info)));
       case kENCODING_NONE:
         return llInt(int64_t(0));
       default:
         CHECK(false);
     }
   }
   switch (type) {
     case kBOOLEAN:
       return llInt(static_cast<int8_t>(inline_int_null_val(type_info)));
     case kTINYINT:
       return llInt(static_cast<int8_t>(inline_int_null_val(type_info)));
     case kSMALLINT:
       return llInt(static_cast<int16_t>(inline_int_null_val(type_info)));
     case kINT:
       return llInt(static_cast<int32_t>(inline_int_null_val(type_info)));
     case kBIGINT:
       return llInt(static_cast<int64_t>(inline_int_null_val(type_info)));
     case kTIME:
     case kTIMESTAMP:
       if (type_info.get_compression() == kENCODING_FIXED) {
         return llInt(inline_fixed_encoding_null_val(type_info));
       }
     case kDATE:
     case kINTERVAL_DAY_TIME:
     case kINTERVAL_YEAR_MONTH:
       return llInt(inline_int_null_val(type_info));
     case kDECIMAL:
     case kNUMERIC:
       return llInt(inline_int_null_val(type_info));
     case kARRAY:
       return llInt(int64_t(0));
     default:
       abort();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::Constant * CgenState::inlineNull ( const SQLTypeInfo & ti )

Definition at line 118 of file CgenState.cpp.

References inlineFpNull(), inlineIntNull(), and SQLTypeInfo::is_fp().

                                                          {
   return ti.is_fp() ? static_cast<llvm::Constant*>(inlineFpNull(ti))
                     : static_cast<llvm::Constant*>(inlineIntNull(ti));
 }

Here is the call graph for this function:

static size_t CgenState::literalBytes ( const CgenState::LiteralValue & lit )

inlinestatic

Definition at line 474 of file CgenState.h.

Referenced by CodeGenerator::codegenHoistedConstantsLoads(), getOrAddLiteral(), and Executor::serializeLiterals().

                                                                {
     switch (lit.which()) {
       case 0:
         return 1;  // int8_t
       case 1:
         return 2;  // int16_t
       case 2:
         return 4;  // int32_t
       case 3:
         return 8;  // int64_t
       case 4:
         return 4;  // float
       case 5:
         return 8;  // double
       case 6:
         return 4;  // std::pair<std::string, int>
       case 7:
         return 4;  // std::string
       case 8:
         return 4;  // std::vector<double>
       case 9:
         return 4;  // std::vector<int32_t>
       case 10:
         return 4;  // std::vector<int8_t>
       case 11:
         return 4;  // std::pair<std::vector<int8_t>, int>
       default:
         abort();
     }
   }

Here is the caller graph for this function:

llvm::ConstantInt* CgenState::llBool ( const bool v ) const

inline

Definition at line 320 of file CgenState.h.

References context_, and ll_bool().

Referenced by Executor::buildJoinLoops(), CodeGenerator::codegenGeosConstructorCall(), CodeGenerator::codegenGeosPredicateCall(), and CodeGenerator::NullCheckCodegen::NullCheckCodegen().

320 { return ::ll_bool(v, context_); }

CgenState::context_

llvm::LLVMContext & context_

Definition: CgenState.h:439

ll_bool

llvm::ConstantInt * ll_bool(const bool v, llvm::LLVMContext &context)

Definition: IRCodegenUtils.h:153

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::ConstantFP* CgenState::llFp ( const float v ) const

inline

Definition at line 310 of file CgenState.h.

References context_.

Referenced by CodeGenerator::codegenArrayExpr(), CodeGenerator::codegenDiv(), CodeGenerator::codegenFpArith(), CodeGenerator::codegenGeoUOper(), CodeGenerator::codegenIsNullNumber(), anonymous_namespace{WindowFunctionIR.cpp}::get_null_value_by_size(), anonymous_namespace{WindowFunctionIR.cpp}::get_null_value_by_size_with_encoding(), and inlineFpNull().

                                             {
     return static_cast<llvm::ConstantFP*>(
         llvm::ConstantFP::get(llvm::Type::getFloatTy(context_), v));
   }

Here is the caller graph for this function:

llvm::ConstantFP* CgenState::llFp ( const double v ) const

inline

Definition at line 315 of file CgenState.h.

References context_.

                                              {
     return static_cast<llvm::ConstantFP*>(
         llvm::ConstantFP::get(llvm::Type::getDoubleTy(context_), v));
   }

template<class T >

llvm::ConstantInt* CgenState::llInt ( const T v ) const

inline

Definition at line 306 of file CgenState.h.

References context_, and ll_int().

                                           {
     return ::ll_int(v, context_);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void CgenState::maybeCloneFunctionRecursive ( llvm::Function * fn )

Definition at line 183 of file CgenState.cpp.

References CHECK, getExecutor(), and vmap_.

Referenced by spatial_type::Transform::codegen(), emitCall(), and emitEntryCall().

                                                             {
   CHECK(fn);
   if (!fn->isDeclaration()) {
     return;
   }
 
   // Get the implementation from the runtime module.
   auto func_impl = getExecutor()->get_rt_module()->getFunction(fn->getName());
   CHECK(func_impl) << fn->getName().str();
 
   if (func_impl->isDeclaration()) {
     return;
   }
 
   auto DestI = fn->arg_begin();
   for (auto arg_it = func_impl->arg_begin(); arg_it != func_impl->arg_end(); ++arg_it) {
     DestI->setName(arg_it->getName());
     vmap_[&*arg_it] = &*DestI++;
   }
 
   llvm::SmallVector<llvm::ReturnInst*, 8> Returns;  // Ignore returns cloned.
 #if LLVM_VERSION_MAJOR > 12
   llvm::CloneFunctionInto(
       fn, func_impl, vmap_, llvm::CloneFunctionChangeType::DifferentModule, Returns);
 #else
   llvm::CloneFunctionInto(fn, func_impl, vmap_, /*ModuleLevelChanges=*/true, Returns);
 #endif
 
   for (auto it = llvm::inst_begin(fn), e = llvm::inst_end(fn); it != e; ++it) {
     if (llvm::isa<llvm::CallInst>(*it)) {
       auto& call = llvm::cast<llvm::CallInst>(*it);
       maybeCloneFunctionRecursive(call.getCalledFunction());
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void CgenState::moveInValuesBitmap ( std::unique_ptr< const InValuesBitmap > & in_values_bitmap )

inline

Definition at line 210 of file CgenState.h.

References in_values_bitmaps_.

                                                                                  {
     if (!in_values_bitmap->isEmpty()) {
       in_values_bitmaps_.emplace_back(std::move(in_values_bitmap));
     }
   }

const StringDictionaryTranslationMgr* CgenState::moveStringDictionaryTranslationMgr ( std::unique_ptr< const StringDictionaryTranslationMgr > && str_dict_translation_mgr )

inline

Definition at line 196 of file CgenState.h.

References str_dict_translation_mgrs_.

Referenced by CodeGenerator::codegen(), CodeGenerator::codegenCastFromString(), and CodeGenerator::codegenPseudoStringOper().

                                                                                       {
     str_dict_translation_mgrs_.emplace_back(std::move(str_dict_translation_mgr));
     return str_dict_translation_mgrs_.back().get();
   }

Here is the caller graph for this function:

void CgenState::replaceFunctionForGpu	(	const std::string &	fcn_to_replace,
		llvm::Function *	fn
	)

Definition at line 331 of file CgenState.cpp.

References run_benchmark_import::args, CHECK, CHECK_EQ, context_, anonymous_namespace{CgenState.cpp}::gpu_replacement_functions, module_, and VLOG.

Referenced by spatial_type::Transform::codegen().

                                                         {
   CHECK(fn);
   CHECK(!fn->isDeclaration());
 
   auto map_it = gpu_replacement_functions.find(fcn_to_replace);
   if (map_it == gpu_replacement_functions.end()) {
     throw QueryMustRunOnCpu("Codegen failed: Could not find replacement functon for " +
                             fcn_to_replace +
                             " to run on gpu. Query step must run in cpu mode.");
   }
   const auto& gpu_fcn_obj = map_it->second;
   CHECK(gpu_fcn_obj);
   VLOG(1) << "Replacing " << fcn_to_replace << " with " << gpu_fcn_obj->name_
           << " for parent function " << fn->getName().str();
 
   for (auto& basic_block : *fn) {
     auto& inst_list = basic_block.getInstList();
     for (auto inst_itr = inst_list.begin(); inst_itr != inst_list.end(); ++inst_itr) {
       if (auto call_inst = llvm::dyn_cast<llvm::CallInst>(inst_itr)) {
         auto called_fcn = call_inst->getCalledFunction();
         CHECK(called_fcn);
 
         if (called_fcn->getName() == fcn_to_replace) {
           std::vector<llvm::Value*> args;
           std::vector<llvm::Type*> arg_types;
           for (auto& arg : call_inst->args()) {
             arg_types.push_back(arg.get()->getType());
             args.push_back(arg.get());
           }
           auto gpu_func = gpu_fcn_obj->getFunction(module_, context_);
           CHECK(gpu_func);
           auto gpu_func_type = gpu_func.getFunctionType();
           CHECK(gpu_func_type);
           CHECK_EQ(gpu_func_type->getReturnType(), called_fcn->getReturnType());
           llvm::ReplaceInstWithInst(call_inst,
                                     llvm::CallInst::Create(gpu_func, args, ""));
           return;
         }
       }
     }
   }
 }

Here is the caller graph for this function:

void CgenState::set_module_shallow_copy	(	const std::unique_ptr< llvm::Module > &	module,
		bool	always_clone = `false`
	)

Definition at line 384 of file CgenState.cpp.

References CodeGenerator::alwaysCloneRuntimeFunction(), module_, and vmap_.

Referenced by GpuReductionHelperJIT::codegen().

                                                            {
   module_ =
       llvm::CloneModule(*llvm_module, vmap_, [always_clone](const llvm::GlobalValue* gv) {
         auto func = llvm::dyn_cast<llvm::Function>(gv);
         if (!func) {
           return true;
         }
         return (func->getLinkage() == llvm::GlobalValue::LinkageTypes::PrivateLinkage ||
                 func->getLinkage() == llvm::GlobalValue::LinkageTypes::InternalLinkage ||
                 (always_clone && CodeGenerator::alwaysCloneRuntimeFunction(func)));
       }).release();
 }