NativeCodegen.cpp File Reference

#include "CodeGenerator.h"
#include "Execute.h"
#include "ExtensionFunctionsWhitelist.h"
#include "GpuSharedMemoryUtils.h"
#include "LLVMFunctionAttributesUtil.h"
#include "OutputBufferInitialization.h"
#include "QueryTemplateGenerator.h"
#include "OSDependent/omnisci_path.h"
#include "Shared/MathUtils.h"
#include "StreamingTopN.h"
#include <llvm/Bitcode/BitcodeReader.h>
#include <llvm/Bitcode/BitcodeWriter.h>
#include <llvm/ExecutionEngine/MCJIT.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/GlobalValue.h>
#include <llvm/IR/InstIterator.h>
#include <llvm/IR/LegacyPassManager.h>
#include <llvm/IR/Verifier.h>
#include <llvm/IRReader/IRReader.h>
#include <llvm/Support/Casting.h>
#include <llvm/Support/FileSystem.h>
#include <llvm/Support/FormattedStream.h>
#include <llvm/Support/MemoryBuffer.h>
#include <llvm/Support/SourceMgr.h>
#include <llvm/Support/TargetRegistry.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Support/raw_os_ostream.h>
#include <llvm/Transforms/IPO.h>
#include <llvm/Transforms/IPO/AlwaysInliner.h>
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
#include <llvm/Transforms/InstCombine/InstCombine.h>
#include <llvm/Transforms/Instrumentation.h>
#include <llvm/Transforms/Scalar.h>
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#include <llvm/Transforms/Utils/Cloning.h>
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include <llvm/Linker/Linker.h>
#include <llvm/Support/raw_ostream.h>

Include dependency graph for NativeCodegen.cpp:

Go to the source code of this file.

Namespaces
	anonymous_namespace{NativeCodegen.cpp}

Functions
void	anonymous_namespace{NativeCodegen.cpp}::eliminate_dead_self_recursive_funcs (llvm::Module &M, const std::unordered_set< llvm::Function *> &live_funcs)
void	anonymous_namespace{NativeCodegen.cpp}::optimize_ir (llvm::Function *query_func, llvm::Module *module, llvm::legacy::PassManager &pass_manager, const std::unordered_set< llvm::Function *> &live_funcs, const CompilationOptions &co)
void	verify_function_ir (const llvm::Function *func)
std::string	anonymous_namespace{NativeCodegen.cpp}::assemblyForCPU (ExecutionEngineWrapper &execution_engine, llvm::Module *module)
std::string	anonymous_namespace{NativeCodegen.cpp}::cpp_to_llvm_name (const std::string &s)
std::string	anonymous_namespace{NativeCodegen.cpp}::gen_array_any_all_sigs ()
std::string	anonymous_namespace{NativeCodegen.cpp}::gen_translate_null_key_sigs ()
llvm::StringRef	get_gpu_target_triple_string ()
llvm::StringRef	get_gpu_data_layout ()
std::map< std::string, std::string >	get_device_parameters (bool cpu_only)
llvm::Module *	read_template_module (llvm::LLVMContext &context)
void	anonymous_namespace{NativeCodegen.cpp}::bind_pos_placeholders (const std::string &pos_fn_name, const bool use_resume_param, llvm::Function *query_func, llvm::Module *module)
void	anonymous_namespace{NativeCodegen.cpp}::set_row_func_argnames (llvm::Function *row_func, const size_t in_col_count, const size_t agg_col_count, const bool hoist_literals)
llvm::Function *	anonymous_namespace{NativeCodegen.cpp}::create_row_function (const size_t in_col_count, const size_t agg_col_count, const bool hoist_literals, llvm::Module *module, llvm::LLVMContext &context)
void	anonymous_namespace{NativeCodegen.cpp}::bind_query (llvm::Function *query_func, const std::string &query_fname, llvm::Function *multifrag_query_func, llvm::Module *module)
std::vector< std::string >	anonymous_namespace{NativeCodegen.cpp}::get_agg_fnames (const std::vector< Analyzer::Expr *> &target_exprs, const bool is_group_by)
std::unique_ptr< llvm::Module >	g_rt_module (read_template_module(getGlobalLLVMContext()))
bool	is_udf_module_present (bool cpu_only)
bool	is_rt_udf_module_present (bool cpu_only)
void	throw_parseIR_error (const llvm::SMDiagnostic &parse_error, std::string src="")
void	read_udf_gpu_module (const std::string &udf_ir_filename)
void	read_udf_cpu_module (const std::string &udf_ir_filename)
void	read_rt_udf_gpu_module (const std::string &udf_ir_string)
void	read_rt_udf_cpu_module (const std::string &udf_ir_string)
template<typename InstType >
llvm::Value *	anonymous_namespace{NativeCodegen.cpp}::find_variable_in_basic_block (llvm::Function *func, std::string bb_name, std::string variable_name)
size_t	anonymous_namespace{NativeCodegen.cpp}::get_shared_memory_size (const bool shared_mem_used, const QueryMemoryDescriptor *query_mem_desc_ptr)
bool	anonymous_namespace{NativeCodegen.cpp}::is_gpu_shared_mem_supported (const QueryMemoryDescriptor *query_mem_desc_ptr, const RelAlgExecutionUnit &ra_exe_unit, const CudaMgr_Namespace::CudaMgr *cuda_mgr, const ExecutorDeviceType device_type, const unsigned gpu_blocksize, const unsigned num_blocks_per_mp)
std::string	anonymous_namespace{NativeCodegen.cpp}::serialize_llvm_metadata_footnotes (llvm::Function *query_func, CgenState *cgen_state)
std::unique_ptr< llvm::Module >	runtime_module_shallow_copy (CgenState *cgen_state)
std::vector< llvm::Value * >	generate_column_heads_load (const int num_columns, llvm::Value *byte_stream_arg, llvm::IRBuilder<> &ir_builder, llvm::LLVMContext &ctx)

Variables
float	g_fraction_code_cache_to_evict = 0.2
std::unique_ptr< llvm::Module >	udf_gpu_module
std::unique_ptr< llvm::Module >	udf_cpu_module
std::unique_ptr< llvm::Module >	rt_udf_gpu_module
std::unique_ptr< llvm::Module >	rt_udf_cpu_module
std::unique_ptr< llvm::Module >	g_rt_module
const std::string	anonymous_namespace{NativeCodegen.cpp}::cuda_rt_decls

Function Documentation

g_rt_module()

std::unique_ptr<llvm::Module> g_rt_module

(

read_template_module(getGlobalLLVMContext())

)

generate_column_heads_load()

std::vector<llvm::Value*> generate_column_heads_load	(	const int	num_columns,
		llvm::Value *	byte_stream_arg,
		llvm::IRBuilder<> &	ir_builder,
		llvm::LLVMContext &	ctx
	)

Loads individual columns from a single, packed pointers buffer (the byte stream arg)

Definition at line 2623 of file NativeCodegen.cpp.

References CHECK.

Referenced by Executor::compileWorkUnit(), and TableFunctionCompilationContext::generateEntryPoint().

                                                                           {
  CHECK(byte_stream_arg);
  const auto max_col_local_id = num_columns - 1;

  std::vector<llvm::Value*> col_heads;
  for (int col_id = 0; col_id <= max_col_local_id; ++col_id) {
    col_heads.emplace_back(ir_builder.CreateLoad(ir_builder.CreateGEP(
        byte_stream_arg, llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), col_id))));
  }
  return col_heads;
}

Here is the caller graph for this function:

get_device_parameters()

std::map<std::string, std::string> get_device_parameters

(

bool

cpu_only

)

Definition at line 708 of file NativeCodegen.cpp.

References checkCudaErrors(), cpu_threads(), get_gpu_data_layout(), get_gpu_target_triple_string(), run_benchmark_import::result, and to_string().

Referenced by DBHandler::max_bytes_for_thrift().

                                                                  {
  std::map<std::string, std::string> result;

  result.insert(std::make_pair("cpu_name", llvm::sys::getHostCPUName()));
  result.insert(std::make_pair("cpu_triple", llvm::sys::getProcessTriple()));
  result.insert(
      std::make_pair("cpu_cores", std::to_string(llvm::sys::getHostNumPhysicalCores())));
  result.insert(std::make_pair("cpu_threads", std::to_string(cpu_threads())));

  llvm::StringMap<bool> cpu_features;
  if (llvm::sys::getHostCPUFeatures(cpu_features)) {
    std::string features_str = "";
    for (auto it = cpu_features.begin(); it != cpu_features.end(); ++it) {
      features_str += (it->getValue() ? " +" : " -");
      features_str += it->getKey().str();
    }
    result.insert(std::make_pair("cpu_features", features_str));
  }

#ifdef HAVE_CUDA
  if (!cpu_only) {
    int device_count = 0;
    checkCudaErrors(cuDeviceGetCount(&device_count));
    if (device_count) {
      CUdevice device{};
      char device_name[256];
      int major = 0, minor = 0;
      checkCudaErrors(cuDeviceGet(&device, 0));  // assuming homogeneous multi-GPU system
      checkCudaErrors(cuDeviceGetName(device_name, 256, device));
      checkCudaErrors(cuDeviceGetAttribute(
          &major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device));
      checkCudaErrors(cuDeviceGetAttribute(
          &minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device));

      result.insert(std::make_pair("gpu_name", device_name));
      result.insert(std::make_pair("gpu_count", std::to_string(device_count)));
      result.insert(std::make_pair("gpu_compute_capability",
                                   std::to_string(major) + "." + std::to_string(minor)));
      result.insert(std::make_pair("gpu_triple", get_gpu_target_triple_string()));
      result.insert(std::make_pair("gpu_datalayout", get_gpu_data_layout()));
    }
  }
#endif

  return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

get_gpu_data_layout()

llvm::StringRef get_gpu_data_layout

(

)

Definition at line 700 of file NativeCodegen.cpp.

Referenced by Executor::compileWorkUnit(), and get_device_parameters().

                                    {
  return llvm::StringRef(
      "e-p:64:64:64-i1:8:8-i8:8:8-"
      "i16:16:16-i32:32:32-i64:64:64-"
      "f32:32:32-f64:64:64-v16:16:16-"
      "v32:32:32-v64:64:64-v128:128:128-n16:32:64");
}

Here is the caller graph for this function:

get_gpu_target_triple_string()

llvm::StringRef get_gpu_target_triple_string

(

)

Definition at line 696 of file NativeCodegen.cpp.

Referenced by Executor::compileWorkUnit(), and get_device_parameters().

                                             {
  return llvm::StringRef("nvptx64-nvidia-cuda");
}

Here is the caller graph for this function:

is_rt_udf_module_present()

bool is_rt_udf_module_present

(

bool

cpu_only

)

Definition at line 1367 of file NativeCodegen.cpp.

References rt_udf_cpu_module, and rt_udf_gpu_module.

Referenced by Executor::compileWorkUnit(), and CodeGenerator::generateNativeGPUCode().

                                             {
  return (cpu_only || rt_udf_gpu_module != nullptr) && (rt_udf_cpu_module != nullptr);
}

Here is the caller graph for this function:

is_udf_module_present()

bool is_udf_module_present

(

bool

cpu_only

)

Definition at line 1363 of file NativeCodegen.cpp.

References udf_cpu_module, and udf_gpu_module.

Referenced by Executor::compileWorkUnit(), and CodeGenerator::generateNativeGPUCode().

                                          {
  return (cpu_only || udf_gpu_module != nullptr) && (udf_cpu_module != nullptr);
}

Here is the caller graph for this function:

read_rt_udf_cpu_module()

void read_rt_udf_cpu_module

(

const std::string &

udf_ir_string

)

Definition at line 1412 of file NativeCodegen.cpp.

References getGlobalLLVMContext(), rt_udf_cpu_module, and throw_parseIR_error().

Referenced by DBHandler::register_runtime_extension_functions().

                                                            {
  llvm::SMDiagnostic parse_error;

  auto buf =
      std::make_unique<llvm::MemoryBufferRef>(udf_ir_string, "Runtime UDF for CPU");

  rt_udf_cpu_module = llvm::parseIR(*buf, parse_error, getGlobalLLVMContext());
  if (!rt_udf_cpu_module) {
    throw_parseIR_error(parse_error);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_rt_udf_gpu_module()

void read_rt_udf_gpu_module

(

const std::string &

udf_ir_string

)

Definition at line 1400 of file NativeCodegen.cpp.

References getGlobalLLVMContext(), rt_udf_gpu_module, and throw_parseIR_error().

Referenced by DBHandler::register_runtime_extension_functions().

                                                            {
  llvm::SMDiagnostic parse_error;

  auto buf =
      std::make_unique<llvm::MemoryBufferRef>(udf_ir_string, "Runtime UDF for GPU");

  rt_udf_gpu_module = llvm::parseIR(*buf, parse_error, getGlobalLLVMContext());
  if (!rt_udf_gpu_module) {
    throw_parseIR_error(parse_error);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_template_module()

llvm::Module* read_template_module

(

llvm::LLVMContext &

context

)

Definition at line 1056 of file NativeCodegen.cpp.

References CHECK, and omnisci::get_root_abs_path().

Referenced by anonymous_namespace{NativeCodegen.cpp}::get_agg_fnames(), and ll_bool().

                                                           {
  llvm::SMDiagnostic err;

  auto buffer_or_error = llvm::MemoryBuffer::getFile(omnisci::get_root_abs_path() +
                                                     "/QueryEngine/RuntimeFunctions.bc");
  CHECK(!buffer_or_error.getError());
  llvm::MemoryBuffer* buffer = buffer_or_error.get().get();

  auto owner = llvm::parseBitcodeFile(buffer->getMemBufferRef(), context);
  CHECK(!owner.takeError());
  auto module = owner.get().release();
  CHECK(module);

  return module;
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_udf_cpu_module()

void read_udf_cpu_module

(

const std::string &

udf_ir_filename

)

Definition at line 1389 of file NativeCodegen.cpp.

References getGlobalLLVMContext(), throw_parseIR_error(), and udf_cpu_module.

Referenced by UdfCompiler::readCpuCompiledModule().

                                                           {
  llvm::SMDiagnostic parse_error;

  llvm::StringRef file_name_arg(udf_ir_filename);

  udf_cpu_module = llvm::parseIRFile(file_name_arg, parse_error, getGlobalLLVMContext());
  if (!udf_cpu_module) {
    throw_parseIR_error(parse_error, udf_ir_filename);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_udf_gpu_module()

void read_udf_gpu_module

(

const std::string &

udf_ir_filename

)

Definition at line 1378 of file NativeCodegen.cpp.

References getGlobalLLVMContext(), throw_parseIR_error(), and udf_gpu_module.

Referenced by UdfCompiler::readGpuCompiledModule().

                                                           {
  llvm::SMDiagnostic parse_error;

  llvm::StringRef file_name_arg(udf_ir_filename);

  udf_gpu_module = llvm::parseIRFile(file_name_arg, parse_error, getGlobalLLVMContext());
  if (!udf_gpu_module) {
    throw_parseIR_error(parse_error, udf_ir_filename);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

runtime_module_shallow_copy()

std::unique_ptr<llvm::Module> runtime_module_shallow_copy

(

CgenState *

cgen_state

)

Makes a shallow copy (just declarations) of the runtime module. Function definitions are cloned only if they're used from the generated code.

Definition at line 2605 of file NativeCodegen.cpp.

References CgenState::vmap_.

Referenced by ResultSetReductionJIT::codegen(), GpuReductionHelperJIT::codegen(), StubGenerator::generateStub(), and TableFunctionCompilationContext::TableFunctionCompilationContext().

                                                                             {
  return llvm::CloneModule(
#if LLVM_VERSION_MAJOR >= 7
      *g_rt_module.get(),
#else
      g_rt_module.get(),
#endif
      cgen_state->vmap_,
      [](const llvm::GlobalValue* gv) {
        auto func = llvm::dyn_cast<llvm::Function>(gv);
        if (!func) {
          return true;
        }
        return (func->getLinkage() == llvm::GlobalValue::LinkageTypes::PrivateLinkage ||
                func->getLinkage() == llvm::GlobalValue::LinkageTypes::InternalLinkage);
      });
}

Here is the caller graph for this function:

throw_parseIR_error()

void throw_parseIR_error	(	const llvm::SMDiagnostic &	parse_error,
		std::string	src = ""
	)

Definition at line 1371 of file NativeCodegen.cpp.

References src.

Referenced by read_rt_udf_cpu_module(), read_rt_udf_gpu_module(), read_udf_cpu_module(), and read_udf_gpu_module().

                                                                                  {
  std::string excname = "LLVM IR ParseError: ";
  llvm::raw_string_ostream ss(excname);
  parse_error.print(src.c_str(), ss, false, false);
  throw std::runtime_error(ss.str());
}

Here is the caller graph for this function:

verify_function_ir()

void verify_function_ir

(

const llvm::Function *

func

)

Definition at line 204 of file NativeCodegen.cpp.

References logger::FATAL, and LOG.

Referenced by GpuSharedMemCodeBuilder::codegen(), Executor::compileWorkUnit(), TableFunctionCompilationContext::generateEntryPoint(), StubGenerator::generateStub(), serialize_llvm_object(), and translate_function().

                                                  {
  std::stringstream err_ss;
  llvm::raw_os_ostream err_os(err_ss);
  if (llvm::verifyFunction(*func, &err_os)) {
    func->print(llvm::outs());
    LOG(FATAL) << err_ss.str();
  }
}

Here is the caller graph for this function:

Variable Documentation

g_fraction_code_cache_to_evict

float g_fraction_code_cache_to_evict = 0.2

Definition at line 70 of file NativeCodegen.cpp.

Referenced by CommandLineOptions::fillAdvancedOptions(), and Executor::optimizeAndCodegenGPU().

g_rt_module

std::unique_ptr<llvm::Module> g_rt_module

Referenced by anonymous_namespace{NativeCodegen.cpp}::get_agg_fnames().

rt_udf_cpu_module

std::unique_ptr<llvm::Module> rt_udf_cpu_module

Definition at line 75 of file NativeCodegen.cpp.

Referenced by Executor::compileWorkUnit(), TableFunctionCompilationContext::finalize(), is_rt_udf_module_present(), and read_rt_udf_cpu_module().

rt_udf_gpu_module

std::unique_ptr<llvm::Module> rt_udf_gpu_module

Definition at line 74 of file NativeCodegen.cpp.

Referenced by Executor::compileWorkUnit(), CodeGenerator::generateNativeGPUCode(), is_rt_udf_module_present(), and read_rt_udf_gpu_module().

udf_cpu_module

std::unique_ptr<llvm::Module> udf_cpu_module

Definition at line 73 of file NativeCodegen.cpp.

Referenced by Executor::compileWorkUnit(), is_udf_module_present(), and read_udf_cpu_module().

udf_gpu_module

std::unique_ptr<llvm::Module> udf_gpu_module

Definition at line 72 of file NativeCodegen.cpp.

Referenced by Executor::compileWorkUnit(), CodeGenerator::generateNativeGPUCode(), is_udf_module_present(), and read_udf_gpu_module().