UdfCompiler Class Reference

#include <UDFCompiler.h>

Collaboration diagram for UdfCompiler:

Public Member Functions
	UdfCompiler (const std::string &udf_file_name, CudaMgr_Namespace::NvidiaDeviceArch target_arch, const std::string &clang_path="")
	UdfCompiler (const std::string &udf_file_name, CudaMgr_Namespace::NvidiaDeviceArch target_arch, const std::string &clang_path, const std::vector< std::string > clang_options)
int	compileUdf ()
const std::string &	getAstFileName () const

Private Member Functions
void	init (const std::string &clang_path)
std::string	removeFileExtension (const std::string &path)
std::string	getFileExt (std::string &s)
int	parseToAst (const char *file_name)
std::string	genGpuIrFilename (const char *udf_file_name)
std::string	genCpuIrFilename (const char *udf_file_name)
int	compileToGpuByteCode (const char *udf_file_name, bool cpu_mode)
int	compileToCpuByteCode (const char *udf_file_name)
void	replaceExtn (std::string &s, const std::string &new_ext)
int	compileFromCommandLine (const std::vector< std::string > &command_line)
void	readCompiledModules ()
void	readGpuCompiledModule ()
void	readCpuCompiledModule ()
int	compileForGpu ()

Private Attributes
std::string	udf_file_name_
std::string	udf_ast_file_name_
CudaMgr_Namespace::NvidiaDeviceArch	target_arch_
std::string	clang_path_
std::vector< std::string >	clang_options_

Detailed Description

Definition at line 50 of file UDFCompiler.h.

Constructor & Destructor Documentation

UdfCompiler() [1/2]

UdfCompiler::UdfCompiler	(	const std::string &	udf_file_name,
		CudaMgr_Namespace::NvidiaDeviceArch	target_arch,
		const std::string &	clang_path = ""
	)

Definition at line 378 of file UDFCompiler.cpp.

References init().

    : udf_file_name_(file_name)
    , udf_ast_file_name_(file_name)
    , target_arch_(target_arch) {
  init(clang_path);
}

Here is the call graph for this function:

UdfCompiler() [2/2]

UdfCompiler::UdfCompiler	(	const std::string &	udf_file_name,
		CudaMgr_Namespace::NvidiaDeviceArch	target_arch,
		const std::string &	clang_path,
		const std::vector< std::string >	clang_options
	)

Definition at line 387 of file UDFCompiler.cpp.

References init().

    : udf_file_name_(file_name)
    , udf_ast_file_name_(file_name)
    , target_arch_(target_arch)
    , clang_options_(clang_options) {
  init(clang_path);
}

Here is the call graph for this function:

Member Function Documentation

compileForGpu()

int UdfCompiler::compileForGpu ( )

private

Definition at line 419 of file UDFCompiler.cpp.

References compileToGpuByteCode(), and udf_file_name_.

Referenced by compileUdf().

                               {
  int gpu_compile_result = 1;

  gpu_compile_result = compileToGpuByteCode(udf_file_name_.c_str(), false);

  // If gpu compilation fails but cpu compilation has succeeded, try compiling
  // for the cpu with the assumption the user does not have the CUDA toolkit
  // installed
  //
  // Update: while this approach may work for some cases, it will not
  // work in general as evidenced by the current UdfTest using arrays:
  // generation of PTX will fail. Hence, read_udf_gpu_module is now
  // rejecting LLVM IR with a non-nvptx target triple. However, we
  // will still try cpu compilation but with the aim of detecting any
  // code errors.
  if (gpu_compile_result != 0) {
    gpu_compile_result = compileToGpuByteCode(udf_file_name_.c_str(), true);
  }

  return gpu_compile_result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

compileFromCommandLine()

int UdfCompiler::compileFromCommandLine ( const std::vector< std::string > &  command_line )

private

Definition at line 214 of file UDFCompiler.cpp.

References parse_ast::end, logger::FATAL, UdfClangDriver::getClangDriver(), LOG, run_benchmark_import::res, and UdfClangDriver::the_driver.

                                                                                {
  UdfClangDriver compiler_driver(clang_path_);
  auto the_driver(compiler_driver.getClangDriver());

  std::vector<const char*> clang_command_opts;
  clang_command_opts.reserve(command_line.size() + clang_options_.size());
  // add required options first
  std::transform(std::begin(command_line),
                 std::end(command_line),
                 std::back_inserter(clang_command_opts),
                 [&](const std::string& str) { return str.c_str(); });

  // If there were additional clang options passed to the system, append them here
  if (!clang_options_.empty()) {
    std::transform(std::begin(clang_options_),
                   std::end(clang_options_),
                   std::back_inserter(clang_command_opts),
                   [&](const std::string& str) { return str.c_str(); });
  }

  the_driver->CCPrintOptions = 0;
  std::unique_ptr<driver::Compilation> compilation(
      the_driver->BuildCompilation(clang_command_opts));

  if (!compilation) {
    LOG(FATAL) << "failed to build compilation object!\n";
  }

  llvm::SmallVector<std::pair<int, const driver::Command*>, 10> failing_commands;
  int res = the_driver->ExecuteCompilation(*compilation, failing_commands);
  if (res < 0) {
    for (const std::pair<int, const driver::Command*>& p : failing_commands) {
      if (p.first) {
        the_driver->generateCompilationDiagnostics(*compilation, *p.second);
      }
    }
  }

  return res;
}

Here is the call graph for this function:

compileToCpuByteCode()

int UdfCompiler::compileToCpuByteCode ( const char *  udf_file_name )

private

Definition at line 297 of file UDFCompiler.cpp.

Referenced by compileUdf().

                                                               {
  std::string cpu_out_filename(genCpuIrFilename(udf_file_name));

  std::vector<std::string> command_line{clang_path_,
                                        "-c",
                                        "-O2",
                                        "-emit-llvm",
                                        "-o",
                                        cpu_out_filename,
                                        "-std=c++14",
                                        "-DNO_BOOST",
                                        udf_file_name};
  return compileFromCommandLine(command_line);
}

Here is the caller graph for this function:

compileToGpuByteCode()

int UdfCompiler::compileToGpuByteCode ( const char *  udf_file_name, bool  cpu_mode  )

private

Definition at line 255 of file UDFCompiler.cpp.

References CudaMgr_Namespace::CudaMgr::deviceArchToSM(), and get_cuda_home().

Referenced by compileForGpu().

                                                                              {
  std::string gpu_out_filename(genGpuIrFilename(udf_file_name));

  std::vector<std::string> command_line{clang_path_,
                                        "-c",
                                        "-O2",
                                        "-emit-llvm",
                                        "-o",
                                        gpu_out_filename,
                                        "-std=c++14",
                                        "-DNO_BOOST"};

  // If we are not compiling for cpu mode, then target the gpu
  // Otherwise assume we can generic ir that will
  // be translated to gpu code during target code generation
#ifdef HAVE_CUDA
  if (!cpu_mode) {
    command_line.emplace_back("--cuda-gpu-arch=" +
                              CudaMgr_Namespace::CudaMgr::deviceArchToSM(target_arch_));
    command_line.emplace_back("--cuda-device-only");
    command_line.emplace_back("-xcuda");
    command_line.emplace_back("--no-cuda-version-check");
    const auto cuda_path = get_cuda_home();
    if (cuda_path != "") {
      command_line.emplace_back("--cuda-path=" + cuda_path);
    }
  }
#endif

  command_line.emplace_back(udf_file_name);

  // clean up from previous runs
  boost::filesystem::remove(gpu_out_filename);
  auto status = compileFromCommandLine(command_line);
  // make sure that compilation actually succeeded by checking the
  // output file:
  if (!status && !boost::filesystem::exists(gpu_out_filename)) {
    status = 2;
  }
  return status;
}

Here is the call graph for this function:

Here is the caller graph for this function:

compileUdf()

int UdfCompiler::compileUdf

(

)

Definition at line 441 of file UDFCompiler.cpp.

References compileForGpu(), compileToCpuByteCode(), logger::FATAL, logger::INFO, LOG, parseToAst(), readCpuCompiledModule(), readGpuCompiledModule(), and udf_file_name_.

Referenced by DBHandler::DBHandler().

                            {
  LOG(INFO) << "UDFCompiler filename to compile: " << udf_file_name_;
  if (!boost::filesystem::exists(udf_file_name_)) {
    LOG(FATAL) << "User defined function file " << udf_file_name_ << " does not exist.";
    return 1;
  }

  auto ast_result = parseToAst(udf_file_name_.c_str());

  if (ast_result == 0) {
    // Compile udf file to generate cpu and gpu bytecode files

    int cpu_compile_result = compileToCpuByteCode(udf_file_name_.c_str());
#ifdef HAVE_CUDA
    int gpu_compile_result = 1;
#endif

    if (cpu_compile_result == 0) {
      readCpuCompiledModule();
#ifdef HAVE_CUDA
      gpu_compile_result = compileForGpu();
      if (gpu_compile_result == 0) {
        readGpuCompiledModule();
      } else {
        LOG(FATAL) << "Unable to compile UDF file for gpu";
        return 1;
      }
#endif
    } else {
      LOG(FATAL) << "Unable to compile UDF file for cpu";
      return 1;
    }
  } else {
    LOG(FATAL) << "Unable to create AST file for udf compilation";
    return 1;
  }

  return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function:

genCpuIrFilename()

std::string UdfCompiler::genCpuIrFilename ( const char *  udf_file_name )

private

Definition at line 207 of file UDFCompiler.cpp.

Referenced by readCpuCompiledModule().

                                                                {
  std::string cpu_file_name(removeFileExtension(udf_fileName));

  cpu_file_name += "_cpu.bc";
  return cpu_file_name;
}

Here is the caller graph for this function:

genGpuIrFilename()

std::string UdfCompiler::genGpuIrFilename ( const char *  udf_file_name )

private

Definition at line 200 of file UDFCompiler.cpp.

Referenced by readGpuCompiledModule().

                                                                 {
  std::string gpu_file_name(removeFileExtension(udf_file_name));

  gpu_file_name += "_gpu.bc";
  return gpu_file_name;
}

Here is the caller graph for this function:

getAstFileName()

const std::string & UdfCompiler::getAstFileName

(

)

const

Definition at line 350 of file UDFCompiler.cpp.

Referenced by DBHandler::DBHandler().

                                                   {
  return udf_ast_file_name_;
}

Here is the caller graph for this function:

getFileExt()

std::string UdfCompiler::getFileExt ( std::string &  s )

private

Definition at line 185 of file UDFCompiler.cpp.

                                              {
  size_t i = s.rfind('.', s.length());
  if (1 != std::string::npos) {
    return (s.substr(i + 1, s.length() - i));
  }
}

init()

void UdfCompiler::init ( const std::string &  clang_path )

private

Definition at line 354 of file UDFCompiler.cpp.

Referenced by UdfCompiler().

                                                  {
  replaceExtn(udf_ast_file_name_, "ast");

  if (clang_path.empty()) {
    clang_path_.assign(llvm::sys::findProgramByName("clang++").get());
    if (clang_path_.empty()) {
      throw std::runtime_error(
          "Unable to find clang++ to compile user defined functions");
    }
  } else {
    clang_path_.assign(clang_path);

    if (!boost::filesystem::exists(clang_path)) {
      throw std::runtime_error("Path provided for udf compiler " + clang_path +
                               " does not exist.");
    }

    if (boost::filesystem::is_directory(clang_path)) {
      throw std::runtime_error("Path provided for udf compiler " + clang_path +
                               " is not to the clang++ executable.");
    }
  }
}

Here is the caller graph for this function:

parseToAst()

int UdfCompiler::parseToAst ( const char *  file_name )

private

Definition at line 312 of file UDFCompiler.cpp.

References anonymous_namespace{UDFCompiler.cpp}::convert(), gpu_enabled::copy(), UdfClangDriver::getClangDriver(), UdfClangDriver::the_driver, and ToolingSampleCategory().

Referenced by compileUdf().

                                                 {
  UdfClangDriver the_driver(clang_path_);
  std::string resource_path = the_driver.getClangDriver()->ResourceDir;
  std::string include_option =
      std::string("-I") + resource_path + std::string("/include");

  std::vector<std::string> arg_vector;
  arg_vector.emplace_back("astparser");
  arg_vector.emplace_back(file_name);
  arg_vector.emplace_back("--");
  arg_vector.emplace_back("-DNO_BOOST");
  arg_vector.emplace_back(include_option);

  if (clang_options_.size() > 0) {
    std::copy(
        clang_options_.begin(), clang_options_.end(), std::back_inserter(arg_vector));
  }

  std::vector<const char*> arg_vec2;
  std::transform(
      arg_vector.begin(), arg_vector.end(), std::back_inserter(arg_vec2), convert);

  int num_args = arg_vec2.size();
  CommonOptionsParser op(num_args, &arg_vec2[0], ToolingSampleCategory);
  ClangTool tool(op.getCompilations(), op.getSourcePathList());

  std::string out_name(file_name);
  std::string file_ext("ast");
  replaceExtn(out_name, file_ext);

  std::error_code out_error_info;
  llvm::raw_fd_ostream out_file(
      llvm::StringRef(out_name), out_error_info, llvm::sys::fs::F_None);

  auto factory = std::make_unique<ToolFactory>(out_file);
  return tool.run(factory.get());
}

Here is the call graph for this function:

Here is the caller graph for this function:

readCompiledModules()

void UdfCompiler::readCompiledModules ( )

private

Definition at line 414 of file UDFCompiler.cpp.

References readCpuCompiledModule(), and readGpuCompiledModule().

                                      {
  readCpuCompiledModule();
  readGpuCompiledModule();
}

Here is the call graph for this function:

readCpuCompiledModule()

void UdfCompiler::readCpuCompiledModule ( )

private

Definition at line 398 of file UDFCompiler.cpp.

References genCpuIrFilename(), read_udf_cpu_module(), udf_file_name_, and VLOG.

Referenced by compileUdf(), and readCompiledModules().

                                        {
  std::string cpu_ir_file(genCpuIrFilename(udf_file_name_.c_str()));

  VLOG(1) << "UDFCompiler cpu bc file = " << cpu_ir_file;

  read_udf_cpu_module(cpu_ir_file);
}

Here is the call graph for this function:

Here is the caller graph for this function:

readGpuCompiledModule()

void UdfCompiler::readGpuCompiledModule ( )

private

Definition at line 406 of file UDFCompiler.cpp.

References genGpuIrFilename(), read_udf_gpu_module(), udf_file_name_, and VLOG.

Referenced by compileUdf(), and readCompiledModules().

                                        {
  std::string gpu_ir_file(genGpuIrFilename(udf_file_name_.c_str()));

  VLOG(1) << "UDFCompiler gpu bc file = " << gpu_ir_file;

  read_udf_gpu_module(gpu_ir_file);
}

Here is the call graph for this function:

Here is the caller graph for this function:

removeFileExtension()

std::string UdfCompiler::removeFileExtension ( const std::string &  path )

private

Definition at line 172 of file UDFCompiler.cpp.

                                                                {
  if (path == "." || path == "..") {
    return path;
  }

  size_t pos = path.find_last_of("\\/.");
  if (pos != std::string::npos && path[pos] == '.') {
    return path.substr(0, pos);
  }

  return path;
}

replaceExtn()

void UdfCompiler::replaceExtn ( std::string &  s, const std::string &  new_ext  )

private

Definition at line 192 of file UDFCompiler.cpp.

                                                                    {
  std::string::size_type i = s.rfind('.', s.length());

  if (i != std::string::npos) {
    s.replace(i + 1, getFileExt(s).length(), new_ext);
  }
}

Member Data Documentation

clang_options_

std::vector<std::string> UdfCompiler::clang_options_

private

Definition at line 83 of file UDFCompiler.h.

clang_path_

std::string UdfCompiler::clang_path_

private

Definition at line 82 of file UDFCompiler.h.

target_arch_

CudaMgr_Namespace::NvidiaDeviceArch UdfCompiler::target_arch_

private

Definition at line 81 of file UDFCompiler.h.

udf_ast_file_name_

std::string UdfCompiler::udf_ast_file_name_

private

Definition at line 80 of file UDFCompiler.h.

udf_file_name_

std::string UdfCompiler::udf_file_name_

private

Definition at line 79 of file UDFCompiler.h.

Referenced by compileForGpu(), compileUdf(), readCpuCompiledModule(), and readGpuCompiledModule().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/UDFCompiler.h
• /home/jenkins-slave/workspace/core-os-doxygen/QueryEngine/UDFCompiler.cpp