OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
NativeCodegen.cpp File Reference
#include "QueryEngine/Execute.h"
#include <llvm/Analysis/ScopedNoAliasAA.h>
#include <llvm/Analysis/TypeBasedAliasAnalysis.h>
#include <llvm/Bitcode/BitcodeReader.h>
#include <llvm/Bitcode/BitcodeWriter.h>
#include <llvm/ExecutionEngine/MCJIT.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/GlobalValue.h>
#include <llvm/IR/InstIterator.h>
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/IR/Intrinsics.h>
#include <llvm/IR/LegacyPassManager.h>
#include <llvm/IR/Verifier.h>
#include <llvm/IRReader/IRReader.h>
#include <llvm/Support/TargetRegistry.h>
#include <llvm/Support/Casting.h>
#include <llvm/Support/FileSystem.h>
#include <llvm/Support/FormattedStream.h>
#include <llvm/Support/MemoryBuffer.h>
#include <llvm/Support/SourceMgr.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Support/raw_os_ostream.h>
#include <llvm/Support/raw_ostream.h>
#include <llvm/Transforms/IPO.h>
#include <llvm/Transforms/IPO/AlwaysInliner.h>
#include <llvm/Transforms/IPO/InferFunctionAttrs.h>
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
#include <llvm/Transforms/InstCombine/InstCombine.h>
#include <llvm/Transforms/Instrumentation.h>
#include <llvm/Transforms/Scalar.h>
#include <llvm/Transforms/Scalar/GVN.h>
#include <llvm/Transforms/Scalar/InstSimplifyPass.h>
#include <llvm/Transforms/Utils.h>
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#include <llvm/Transforms/Utils/Cloning.h>
#include "CudaMgr/CudaMgr.h"
#include "QueryEngine/CodeGenerator.h"
#include "QueryEngine/ExtensionFunctionsWhitelist.h"
#include "QueryEngine/GpuSharedMemoryUtils.h"
#include "QueryEngine/LLVMFunctionAttributesUtil.h"
#include "QueryEngine/Optimization/AnnotateInternalFunctionsPass.h"
#include "QueryEngine/OutputBufferInitialization.h"
#include "QueryEngine/QueryEngine.h"
#include "QueryEngine/QueryTemplateGenerator.h"
#include "Shared/InlineNullValues.h"
#include "Shared/MathUtils.h"
#include "StreamingTopN.h"
+ Include dependency graph for NativeCodegen.cpp:

Go to the source code of this file.

Namespaces

 anonymous_namespace{NativeCodegen.cpp}
 

Macros

#define SHOW_DEFINED(MODULE)
 
#define SHOW_FUNCTIONS(MODULE)
 

Functions

void anonymous_namespace{NativeCodegen.cpp}::throw_parseIR_error (const llvm::SMDiagnostic &parse_error, std::string src="", const bool is_gpu=false)
 
template<typename T = void>
void anonymous_namespace{NativeCodegen.cpp}::show_defined (llvm::Module &llvm_module)
 
template<typename T = void>
void anonymous_namespace{NativeCodegen.cpp}::show_defined (llvm::Module *llvm_module)
 
template<typename T = void>
void anonymous_namespace{NativeCodegen.cpp}::show_defined (std::unique_ptr< llvm::Module > &llvm_module)
 
template<typename T = void>
void anonymous_namespace{NativeCodegen.cpp}::scan_function_calls (llvm::Function &F, std::unordered_set< std::string > &defined, std::unordered_set< std::string > &undefined, const std::unordered_set< std::string > &ignored)
 
template<typename T = void>
void anonymous_namespace{NativeCodegen.cpp}::scan_function_calls (llvm::Module &llvm_module, std::unordered_set< std::string > &defined, std::unordered_set< std::string > &undefined, const std::unordered_set< std::string > &ignored)
 
template<typename T = void>
std::tuple< std::unordered_set
< std::string >
, std::unordered_set
< std::string > > 
anonymous_namespace{NativeCodegen.cpp}::scan_function_calls (llvm::Module &llvm_module, const std::unordered_set< std::string > &ignored={})
 
void anonymous_namespace{NativeCodegen.cpp}::eliminate_dead_self_recursive_funcs (llvm::Module &M, const std::unordered_set< llvm::Function * > &live_funcs)
 
void anonymous_namespace{NativeCodegen.cpp}::optimize_ir (llvm::Function *query_func, llvm::Module *llvm_module, llvm::legacy::PassManager &pass_manager, const std::unordered_set< llvm::Function * > &live_funcs, const bool is_gpu_smem_used, const CompilationOptions &co)
 
void verify_function_ir (const llvm::Function *func)
 
std::string anonymous_namespace{NativeCodegen.cpp}::assemblyForCPU (ExecutionEngineWrapper &execution_engine, llvm::Module *llvm_module)
 
ExecutionEngineWrapper anonymous_namespace{NativeCodegen.cpp}::create_execution_engine (llvm::Module *llvm_module, llvm::EngineBuilder &eb, const CompilationOptions &co)
 
std::string anonymous_namespace{NativeCodegen.cpp}::cpp_to_llvm_name (const std::string &s)
 
std::string anonymous_namespace{NativeCodegen.cpp}::gen_array_any_all_sigs ()
 
std::string anonymous_namespace{NativeCodegen.cpp}::gen_translate_null_key_sigs ()
 
llvm::StringRef get_gpu_target_triple_string ()
 
llvm::StringRef get_gpu_data_layout ()
 
std::map< std::string,
std::string > 
get_device_parameters (bool cpu_only)
 
std::unique_ptr< llvm::Module > read_llvm_module_from_bc_file (const std::string &bc_filename, llvm::LLVMContext &context)
 
std::unique_ptr< llvm::Module > read_llvm_module_from_ir_file (const std::string &udf_ir_filename, llvm::LLVMContext &ctx, bool is_gpu=false)
 
std::unique_ptr< llvm::Module > read_llvm_module_from_ir_string (const std::string &udf_ir_string, llvm::LLVMContext &ctx, bool is_gpu=false)
 
void anonymous_namespace{NativeCodegen.cpp}::bind_pos_placeholders (const std::string &pos_fn_name, const bool use_resume_param, llvm::Function *query_func, llvm::Module *llvm_module)
 
void anonymous_namespace{NativeCodegen.cpp}::set_row_func_argnames (llvm::Function *row_func, const size_t in_col_count, const size_t agg_col_count, const bool hoist_literals)
 
llvm::Function * anonymous_namespace{NativeCodegen.cpp}::create_row_function (const size_t in_col_count, const size_t agg_col_count, const bool hoist_literals, llvm::Module *llvm_module, llvm::LLVMContext &context)
 
void anonymous_namespace{NativeCodegen.cpp}::bind_query (llvm::Function *query_func, const std::string &query_fname, llvm::Function *multifrag_query_func, llvm::Module *llvm_module)
 
std::vector< std::string > anonymous_namespace{NativeCodegen.cpp}::get_agg_fnames (const std::vector< Analyzer::Expr * > &target_exprs, const bool is_group_by)
 
template<typename InstType >
llvm::Value * anonymous_namespace{NativeCodegen.cpp}::find_variable_in_basic_block (llvm::Function *func, std::string bb_name, std::string variable_name)
 
size_t anonymous_namespace{NativeCodegen.cpp}::get_shared_memory_size (const bool shared_mem_used, const QueryMemoryDescriptor *query_mem_desc_ptr)
 
bool anonymous_namespace{NativeCodegen.cpp}::is_gpu_shared_mem_supported (const QueryMemoryDescriptor *query_mem_desc_ptr, const RelAlgExecutionUnit &ra_exe_unit, const CudaMgr_Namespace::CudaMgr *cuda_mgr, const ExecutorDeviceType device_type, const unsigned cuda_blocksize, const unsigned num_blocks_per_mp)
 
std::string anonymous_namespace{NativeCodegen.cpp}::serialize_llvm_metadata_footnotes (llvm::Function *query_func, CgenState *cgen_state)
 
std::vector< llvm::Value * > generate_column_heads_load (const int num_columns, llvm::Value *byte_stream_arg, llvm::IRBuilder<> &ir_builder, llvm::LLVMContext &ctx)
 

Variables

float g_fraction_code_cache_to_evict = 0.2
 
static llvm::sys::Mutex g_ee_create_mutex
 
const std::string anonymous_namespace{NativeCodegen.cpp}::cuda_rt_decls
 

Macro Definition Documentation

#define SHOW_DEFINED (   MODULE)
Value:
{ \
std::cout << __func__ << "#" << __LINE__ << ": " #MODULE << " "; \
::show_defined(MODULE); \
}
void show_defined(llvm::Module &llvm_module)

Definition at line 143 of file NativeCodegen.cpp.

#define SHOW_FUNCTIONS (   MODULE)
Value:
{ \
std::cout << __func__ << "#" << __LINE__ << ": " #MODULE << " "; \
::show_functions(MODULE); \
}

Definition at line 149 of file NativeCodegen.cpp.

Function Documentation

std::vector<llvm::Value*> generate_column_heads_load ( const int  num_columns,
llvm::Value *  byte_stream_arg,
llvm::IRBuilder<> &  ir_builder,
llvm::LLVMContext &  ctx 
)

Loads individual columns from a single, packed pointers buffer (the byte stream arg)

Definition at line 3360 of file NativeCodegen.cpp.

Referenced by TableFunctionCompilationContext::generateEntryPoint().

3363  {
3364  CHECK(byte_stream_arg);
3365  const auto max_col_local_id = num_columns - 1;
3366 
3367  std::vector<llvm::Value*> col_heads;
3368  for (int col_id = 0; col_id <= max_col_local_id; ++col_id) {
3369  auto* gep = ir_builder.CreateGEP(
3370  byte_stream_arg->getType()->getScalarType()->getPointerElementType(),
3371  byte_stream_arg,
3372  llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), col_id));
3373  col_heads.emplace_back(
3374  ir_builder.CreateLoad(gep->getType()->getPointerElementType(), gep));
3375  }
3376  return col_heads;
3377 }
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

std::map<std::string, std::string> get_device_parameters ( bool  cpu_only)

Definition at line 931 of file NativeCodegen.cpp.

931  {
932  std::map<std::string, std::string> result;
933 
934  result.insert(std::make_pair("cpu_name", llvm::sys::getHostCPUName()));
935  result.insert(std::make_pair("cpu_triple", llvm::sys::getProcessTriple()));
936  result.insert(
937  std::make_pair("cpu_cores", std::to_string(llvm::sys::getHostNumPhysicalCores())));
938  result.insert(std::make_pair("cpu_threads", std::to_string(cpu_threads())));
939 
940  // https://en.cppreference.com/w/cpp/language/types
941  std::string sizeof_types;
942  sizeof_types += "bool:" + std::to_string(sizeof(bool)) + ";";
943  sizeof_types += "size_t:" + std::to_string(sizeof(size_t)) + ";";
944  sizeof_types += "ssize_t:" + std::to_string(sizeof(ssize_t)) + ";";
945  sizeof_types += "char:" + std::to_string(sizeof(char)) + ";";
946  sizeof_types += "uchar:" + std::to_string(sizeof(unsigned char)) + ";";
947  sizeof_types += "short:" + std::to_string(sizeof(short)) + ";";
948  sizeof_types += "ushort:" + std::to_string(sizeof(unsigned short int)) + ";";
949  sizeof_types += "int:" + std::to_string(sizeof(int)) + ";";
950  sizeof_types += "uint:" + std::to_string(sizeof(unsigned int)) + ";";
951  sizeof_types += "long:" + std::to_string(sizeof(long int)) + ";";
952  sizeof_types += "ulong:" + std::to_string(sizeof(unsigned long int)) + ";";
953  sizeof_types += "longlong:" + std::to_string(sizeof(long long int)) + ";";
954  sizeof_types += "ulonglong:" + std::to_string(sizeof(unsigned long long int)) + ";";
955  sizeof_types += "float:" + std::to_string(sizeof(float)) + ";";
956  sizeof_types += "double:" + std::to_string(sizeof(double)) + ";";
957  sizeof_types += "longdouble:" + std::to_string(sizeof(long double)) + ";";
958  sizeof_types += "voidptr:" + std::to_string(sizeof(void*)) + ";";
959 
960  result.insert(std::make_pair("type_sizeof", sizeof_types));
961 
962  std::string null_values;
963  null_values += "boolean1:" + std::to_string(serialized_null_value<bool>()) + ";";
964  null_values += "boolean8:" + std::to_string(serialized_null_value<int8_t>()) + ";";
965  null_values += "int8:" + std::to_string(serialized_null_value<int8_t>()) + ";";
966  null_values += "int16:" + std::to_string(serialized_null_value<int16_t>()) + ";";
967  null_values += "int32:" + std::to_string(serialized_null_value<int32_t>()) + ";";
968  null_values += "int64:" + std::to_string(serialized_null_value<int64_t>()) + ";";
969  null_values += "uint8:" + std::to_string(serialized_null_value<uint8_t>()) + ";";
970  null_values += "uint16:" + std::to_string(serialized_null_value<uint16_t>()) + ";";
971  null_values += "uint32:" + std::to_string(serialized_null_value<uint32_t>()) + ";";
972  null_values += "uint64:" + std::to_string(serialized_null_value<uint64_t>()) + ";";
973  null_values += "float32:" + std::to_string(serialized_null_value<float>()) + ";";
974  null_values += "float64:" + std::to_string(serialized_null_value<double>()) + ";";
975  null_values +=
976  "Array<boolean8>:" + std::to_string(serialized_null_value<int8_t, true>()) + ";";
977  null_values +=
978  "Array<int8>:" + std::to_string(serialized_null_value<int8_t, true>()) + ";";
979  null_values +=
980  "Array<int16>:" + std::to_string(serialized_null_value<int16_t, true>()) + ";";
981  null_values +=
982  "Array<int32>:" + std::to_string(serialized_null_value<int32_t, true>()) + ";";
983  null_values +=
984  "Array<int64>:" + std::to_string(serialized_null_value<int64_t, true>()) + ";";
985  null_values +=
986  "Array<float32>:" + std::to_string(serialized_null_value<float, true>()) + ";";
987  null_values +=
988  "Array<float64>:" + std::to_string(serialized_null_value<double, true>()) + ";";
989 
990  result.insert(std::make_pair("null_values", null_values));
991 
992  llvm::StringMap<bool> cpu_features;
993  if (llvm::sys::getHostCPUFeatures(cpu_features)) {
994  std::string features_str = "";
995  for (auto it = cpu_features.begin(); it != cpu_features.end(); ++it) {
996  features_str += (it->getValue() ? " +" : " -");
997  features_str += it->getKey().str();
998  }
999  result.insert(std::make_pair("cpu_features", features_str));
1000  }
1001 
1002  result.insert(std::make_pair("llvm_version",
1003  std::to_string(LLVM_VERSION_MAJOR) + "." +
1004  std::to_string(LLVM_VERSION_MINOR) + "." +
1005  std::to_string(LLVM_VERSION_PATCH)));
1006 
1007 #ifdef HAVE_CUDA
1008  if (!cpu_only) {
1009  int device_count = 0;
1010  checkCudaErrors(cuDeviceGetCount(&device_count));
1011  if (device_count) {
1012  CUdevice device{};
1013  char device_name[256];
1014  int major = 0, minor = 0;
1015  int driver_version;
1016  checkCudaErrors(cuDeviceGet(&device, 0)); // assuming homogeneous multi-GPU system
1017  checkCudaErrors(cuDeviceGetName(device_name, 256, device));
1018  checkCudaErrors(cuDeviceGetAttribute(
1019  &major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device));
1020  checkCudaErrors(cuDeviceGetAttribute(
1021  &minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device));
1022  checkCudaErrors(cuDriverGetVersion(&driver_version));
1023 
1024  result.insert(std::make_pair("gpu_name", device_name));
1025  result.insert(std::make_pair("gpu_count", std::to_string(device_count)));
1026  result.insert(std::make_pair("gpu_compute_capability",
1027  std::to_string(major) + "." + std::to_string(minor)));
1028  result.insert(std::make_pair("gpu_triple", get_gpu_target_triple_string()));
1029  result.insert(std::make_pair("gpu_datalayout", get_gpu_data_layout()));
1030  result.insert(std::make_pair("gpu_driver",
1031  "CUDA " + std::to_string(driver_version / 1000) + "." +
1032  std::to_string((driver_version % 1000) / 10)));
1033 
1034  auto rt_libdevice_path = get_cuda_libdevice_dir() + "/libdevice.10.bc";
1035  result.insert(
1036  std::make_pair("gpu_has_libdevice",
1037  std::to_string(boost::filesystem::exists(rt_libdevice_path))));
1038  }
1039  }
1040 #endif
1041 
1042  return result;
1043 }
std::string get_cuda_libdevice_dir(void)
Definition: CudaMgr.cpp:494
void checkCudaErrors(CUresult err)
Definition: sample.cpp:38
llvm::StringRef get_gpu_data_layout()
std::string to_string(char const *&&v)
llvm::StringRef get_gpu_target_triple_string()
int CUdevice
Definition: nocuda.h:20
int cpu_threads()
Definition: thread_count.h:25
llvm::StringRef get_gpu_data_layout ( )

Definition at line 923 of file NativeCodegen.cpp.

923  {
924  return llvm::StringRef(
925  "e-p:64:64:64-i1:8:8-i8:8:8-"
926  "i16:16:16-i32:32:32-i64:64:64-"
927  "f32:32:32-f64:64:64-v16:16:16-"
928  "v32:32:32-v64:64:64-v128:128:128-n16:32:64");
929 }
llvm::StringRef get_gpu_target_triple_string ( )

Definition at line 919 of file NativeCodegen.cpp.

919  {
920  return llvm::StringRef("nvptx64-nvidia-cuda");
921 }
std::unique_ptr<llvm::Module> read_llvm_module_from_bc_file ( const std::string &  bc_filename,
llvm::LLVMContext &  context 
)

Definition at line 1526 of file NativeCodegen.cpp.

References run_benchmark_import::args.

Referenced by Executor::update_extension_modules().

1528  {
1529  llvm::SMDiagnostic err;
1530 
1531  auto buffer_or_error = llvm::MemoryBuffer::getFile(bc_filename);
1532  CHECK(!buffer_or_error.getError()) << "bc_filename=" << bc_filename;
1533  llvm::MemoryBuffer* buffer = buffer_or_error.get().get();
1534 
1535  auto owner = llvm::parseBitcodeFile(buffer->getMemBufferRef(), context);
1536  CHECK(!owner.takeError());
1537  CHECK(owner->get());
1538  return std::move(owner.get());
1539 }
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

std::unique_ptr<llvm::Module> read_llvm_module_from_ir_file ( const std::string &  udf_ir_filename,
llvm::LLVMContext &  ctx,
bool  is_gpu = false 
)

Definition at line 1541 of file NativeCodegen.cpp.

Referenced by Executor::update_extension_modules().

1544  {
1545  llvm::SMDiagnostic parse_error;
1546 
1547  llvm::StringRef file_name_arg(udf_ir_filename);
1548 
1549  auto owner = llvm::parseIRFile(file_name_arg, parse_error, ctx);
1550  if (!owner) {
1551  throw_parseIR_error(parse_error, udf_ir_filename, is_gpu);
1552  }
1553 
1554  if (is_gpu) {
1555  llvm::Triple gpu_triple(owner->getTargetTriple());
1556  if (!gpu_triple.isNVPTX()) {
1557  LOG(WARNING)
1558  << "Expected triple nvptx64-nvidia-cuda for NVVM IR of loadtime UDFs but got "
1559  << gpu_triple.str() << ". Disabling the NVVM IR module.";
1560  return std::unique_ptr<llvm::Module>();
1561  }
1562  }
1563  return owner;
1564 }
#define LOG(tag)
Definition: Logger.h:285
void throw_parseIR_error(const llvm::SMDiagnostic &parse_error, std::string src="", const bool is_gpu=false)

+ Here is the caller graph for this function:

std::unique_ptr<llvm::Module> read_llvm_module_from_ir_string ( const std::string &  udf_ir_string,
llvm::LLVMContext &  ctx,
bool  is_gpu = false 
)

Definition at line 1566 of file NativeCodegen.cpp.

Referenced by Executor::update_extension_modules().

1569  {
1570  llvm::SMDiagnostic parse_error;
1571 
1572  auto buf = std::make_unique<llvm::MemoryBufferRef>(udf_ir_string,
1573  "Runtime UDF/UDTF LLVM/NVVM IR");
1574 
1575  auto owner = llvm::parseIR(*buf, parse_error, ctx);
1576  if (!owner) {
1577  LOG(IR) << "read_llvm_module_from_ir_string:\n"
1578  << udf_ir_string << "\nEnd of LLVM/NVVM IR";
1579  throw_parseIR_error(parse_error, "", /* is_gpu= */ is_gpu);
1580  }
1581 
1582  if (is_gpu) {
1583  llvm::Triple gpu_triple(owner->getTargetTriple());
1584  if (!gpu_triple.isNVPTX()) {
1585  LOG(IR) << "read_llvm_module_from_ir_string:\n"
1586  << udf_ir_string << "\nEnd of NNVM IR";
1587  LOG(WARNING) << "Expected triple nvptx64-nvidia-cuda for NVVM IR but got "
1588  << gpu_triple.str()
1589  << ". Executing runtime UDF/UDTFs on GPU will be disabled.";
1590  return std::unique_ptr<llvm::Module>();
1591  ;
1592  }
1593  }
1594  return owner;
1595 }
#define LOG(tag)
Definition: Logger.h:285
void throw_parseIR_error(const llvm::SMDiagnostic &parse_error, std::string src="", const bool is_gpu=false)

+ Here is the caller graph for this function:

void verify_function_ir ( const llvm::Function *  func)

Definition at line 388 of file NativeCodegen.cpp.

References logger::FATAL, and LOG.

Referenced by GpuSharedMemCodeBuilder::codegen(), spatial_type::Transform::codegen(), anonymous_namespace{JoinLoopTest.cpp}::create_loop_test_function(), TableFunctionCompilationContext::generateEntryPoint(), StubGenerator::generateStub(), and translate_function().

388  {
389  std::stringstream err_ss;
390  llvm::raw_os_ostream err_os(err_ss);
391  err_os << "\n-----\n";
392  if (llvm::verifyFunction(*func, &err_os)) {
393  err_os << "\n-----\n";
394  func->print(err_os, nullptr);
395  err_os << "\n-----\n";
396  LOG(FATAL) << err_ss.str();
397  }
398 }
#define LOG(tag)
Definition: Logger.h:285

+ Here is the caller graph for this function:

Variable Documentation

llvm::sys::Mutex g_ee_create_mutex
static
float g_fraction_code_cache_to_evict = 0.2

Definition at line 79 of file NativeCodegen.cpp.

Referenced by CommandLineOptions::fillAdvancedOptions().