OmniSciDB  0264ff685a
UdfCompiler Class Reference

#include <UDFCompiler.h>

+ Collaboration diagram for UdfCompiler:

Public Member Functions

 UdfCompiler (const std::string &udf_file_name, CudaMgr_Namespace::NvidiaDeviceArch target_arch, const std::string &clang_path="")
 
 UdfCompiler (const std::string &udf_file_name, CudaMgr_Namespace::NvidiaDeviceArch target_arch, const std::string &clang_path, const std::vector< std::string > clang_options)
 
int compileUdf ()
 
const std::string & getAstFileName () const
 

Private Member Functions

void init (const std::string &clang_path)
 
std::string removeFileExtension (const std::string &path)
 
std::string getFileExt (std::string &s)
 
int parseToAst (const char *file_name)
 
std::string genGpuIrFilename (const char *udf_file_name)
 
std::string genCpuIrFilename (const char *udf_file_name)
 
int compileToGpuByteCode (const char *udf_file_name, bool cpu_mode)
 
int compileToCpuByteCode (const char *udf_file_name)
 
void replaceExtn (std::string &s, const std::string &new_ext)
 
int compileFromCommandLine (const std::vector< std::string > &command_line)
 
void readCompiledModules ()
 
void readGpuCompiledModule ()
 
void readCpuCompiledModule ()
 
int compileForGpu ()
 

Private Attributes

std::string udf_file_name_
 
std::string udf_ast_file_name_
 
CudaMgr_Namespace::NvidiaDeviceArch target_arch_
 
std::string clang_path_
 
std::vector< std::string > clang_options_
 

Detailed Description

Definition at line 50 of file UDFCompiler.h.

Constructor & Destructor Documentation

◆ UdfCompiler() [1/2]

UdfCompiler::UdfCompiler ( const std::string &  udf_file_name,
CudaMgr_Namespace::NvidiaDeviceArch  target_arch,
const std::string &  clang_path = "" 
)

Definition at line 378 of file UDFCompiler.cpp.

References init().

381  : udf_file_name_(file_name)
382  , udf_ast_file_name_(file_name)
383  , target_arch_(target_arch) {
384  init(clang_path);
385 }
std::string udf_ast_file_name_
Definition: UDFCompiler.h:80
CudaMgr_Namespace::NvidiaDeviceArch target_arch_
Definition: UDFCompiler.h:81
void init(const std::string &clang_path)
std::string udf_file_name_
Definition: UDFCompiler.h:79
+ Here is the call graph for this function:

◆ UdfCompiler() [2/2]

UdfCompiler::UdfCompiler ( const std::string &  udf_file_name,
CudaMgr_Namespace::NvidiaDeviceArch  target_arch,
const std::string &  clang_path,
const std::vector< std::string >  clang_options 
)

Definition at line 387 of file UDFCompiler.cpp.

References init().

391  : udf_file_name_(file_name)
392  , udf_ast_file_name_(file_name)
393  , target_arch_(target_arch)
394  , clang_options_(clang_options) {
395  init(clang_path);
396 }
std::vector< std::string > clang_options_
Definition: UDFCompiler.h:83
std::string udf_ast_file_name_
Definition: UDFCompiler.h:80
CudaMgr_Namespace::NvidiaDeviceArch target_arch_
Definition: UDFCompiler.h:81
void init(const std::string &clang_path)
std::string udf_file_name_
Definition: UDFCompiler.h:79
+ Here is the call graph for this function:

Member Function Documentation

◆ compileForGpu()

int UdfCompiler::compileForGpu ( )
private

Definition at line 419 of file UDFCompiler.cpp.

References compileToGpuByteCode(), and udf_file_name_.

Referenced by compileUdf().

419  {
420  int gpu_compile_result = 1;
421 
422  gpu_compile_result = compileToGpuByteCode(udf_file_name_.c_str(), false);
423 
424  // If gpu compilation fails but cpu compilation has succeeded, try compiling
425  // for the cpu with the assumption the user does not have the CUDA toolkit
426  // installed
427  //
428  // Update: while this approach may work for some cases, it will not
429  // work in general as evidenced by the current UdfTest using arrays:
430  // generation of PTX will fail. Hence, read_udf_gpu_module is now
431  // rejecting LLVM IR with a non-nvptx target triple. However, we
432  // will still try cpu compilation but with the aim of detecting any
433  // code errors.
434  if (gpu_compile_result != 0) {
435  gpu_compile_result = compileToGpuByteCode(udf_file_name_.c_str(), true);
436  }
437 
438  return gpu_compile_result;
439 }
int compileToGpuByteCode(const char *udf_file_name, bool cpu_mode)
std::string udf_file_name_
Definition: UDFCompiler.h:79
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ compileFromCommandLine()

int UdfCompiler::compileFromCommandLine ( const std::vector< std::string > &  command_line)
private

Definition at line 214 of file UDFCompiler.cpp.

References parse_ast::end, logger::FATAL, UdfClangDriver::getClangDriver(), LOG, run_benchmark_import::res, and UdfClangDriver::the_driver.

214  {
215  UdfClangDriver compiler_driver(clang_path_);
216  auto the_driver(compiler_driver.getClangDriver());
217 
218  std::vector<const char*> clang_command_opts;
219  clang_command_opts.reserve(command_line.size() + clang_options_.size());
220  // add required options first
221  std::transform(std::begin(command_line),
222  std::end(command_line),
223  std::back_inserter(clang_command_opts),
224  [&](const std::string& str) { return str.c_str(); });
225 
226  // If there were additional clang options passed to the system, append them here
227  if (!clang_options_.empty()) {
228  std::transform(std::begin(clang_options_),
230  std::back_inserter(clang_command_opts),
231  [&](const std::string& str) { return str.c_str(); });
232  }
233 
234  the_driver->CCPrintOptions = 0;
235  std::unique_ptr<driver::Compilation> compilation(
236  the_driver->BuildCompilation(clang_command_opts));
237 
238  if (!compilation) {
239  LOG(FATAL) << "failed to build compilation object!\n";
240  }
241 
242  llvm::SmallVector<std::pair<int, const driver::Command*>, 10> failing_commands;
243  int res = the_driver->ExecuteCompilation(*compilation, failing_commands);
244  if (res < 0) {
245  for (const std::pair<int, const driver::Command*>& p : failing_commands) {
246  if (p.first) {
247  the_driver->generateCompilationDiagnostics(*compilation, *p.second);
248  }
249  }
250  }
251 
252  return res;
253 }
std::vector< std::string > clang_options_
Definition: UDFCompiler.h:83
#define LOG(tag)
Definition: Logger.h:188
std::string clang_path_
Definition: UDFCompiler.h:82
+ Here is the call graph for this function:

◆ compileToCpuByteCode()

int UdfCompiler::compileToCpuByteCode ( const char *  udf_file_name)
private

Definition at line 297 of file UDFCompiler.cpp.

Referenced by compileUdf().

297  {
298  std::string cpu_out_filename(genCpuIrFilename(udf_file_name));
299 
300  std::vector<std::string> command_line{clang_path_,
301  "-c",
302  "-O2",
303  "-emit-llvm",
304  "-o",
305  cpu_out_filename,
306  "-std=c++14",
307  "-DNO_BOOST",
308  udf_file_name};
309  return compileFromCommandLine(command_line);
310 }
std::string genCpuIrFilename(const char *udf_file_name)
int compileFromCommandLine(const std::vector< std::string > &command_line)
std::string clang_path_
Definition: UDFCompiler.h:82
+ Here is the caller graph for this function:

◆ compileToGpuByteCode()

int UdfCompiler::compileToGpuByteCode ( const char *  udf_file_name,
bool  cpu_mode 
)
private

Definition at line 255 of file UDFCompiler.cpp.

References CudaMgr_Namespace::CudaMgr::deviceArchToSM(), and get_cuda_home().

Referenced by compileForGpu().

255  {
256  std::string gpu_out_filename(genGpuIrFilename(udf_file_name));
257 
258  std::vector<std::string> command_line{clang_path_,
259  "-c",
260  "-O2",
261  "-emit-llvm",
262  "-o",
263  gpu_out_filename,
264  "-std=c++14",
265  "-DNO_BOOST"};
266 
267  // If we are not compiling for cpu mode, then target the gpu
268  // Otherwise assume we can generic ir that will
269  // be translated to gpu code during target code generation
270 #ifdef HAVE_CUDA
271  if (!cpu_mode) {
272  command_line.emplace_back("--cuda-gpu-arch=" +
274  command_line.emplace_back("--cuda-device-only");
275  command_line.emplace_back("-xcuda");
276  command_line.emplace_back("--no-cuda-version-check");
277  const auto cuda_path = get_cuda_home();
278  if (cuda_path != "") {
279  command_line.emplace_back("--cuda-path=" + cuda_path);
280  }
281  }
282 #endif
283 
284  command_line.emplace_back(udf_file_name);
285 
286  // clean up from previous runs
287  boost::filesystem::remove(gpu_out_filename);
288  auto status = compileFromCommandLine(command_line);
289  // make sure that compilation actually succeeded by checking the
290  // output file:
291  if (!status && !boost::filesystem::exists(gpu_out_filename)) {
292  status = 2;
293  }
294  return status;
295 }
std::string genGpuIrFilename(const char *udf_file_name)
std::string get_cuda_home(void)
Definition: CudaMgr.cpp:405
CudaMgr_Namespace::NvidiaDeviceArch target_arch_
Definition: UDFCompiler.h:81
static std::string deviceArchToSM(const NvidiaDeviceArch arch)
Definition: CudaMgr.h:148
int compileFromCommandLine(const std::vector< std::string > &command_line)
std::string clang_path_
Definition: UDFCompiler.h:82
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ compileUdf()

int UdfCompiler::compileUdf ( )

Definition at line 441 of file UDFCompiler.cpp.

References compileForGpu(), compileToCpuByteCode(), logger::FATAL, logger::INFO, LOG, parseToAst(), readCpuCompiledModule(), readGpuCompiledModule(), and udf_file_name_.

Referenced by DBHandler::DBHandler().

441  {
442  LOG(INFO) << "UDFCompiler filename to compile: " << udf_file_name_;
443  if (!boost::filesystem::exists(udf_file_name_)) {
444  LOG(FATAL) << "User defined function file " << udf_file_name_ << " does not exist.";
445  return 1;
446  }
447 
448  auto ast_result = parseToAst(udf_file_name_.c_str());
449 
450  if (ast_result == 0) {
451  // Compile udf file to generate cpu and gpu bytecode files
452 
453  int cpu_compile_result = compileToCpuByteCode(udf_file_name_.c_str());
454 #ifdef HAVE_CUDA
455  int gpu_compile_result = 1;
456 #endif
457 
458  if (cpu_compile_result == 0) {
460 #ifdef HAVE_CUDA
461  gpu_compile_result = compileForGpu();
462  if (gpu_compile_result == 0) {
464  } else {
465  LOG(FATAL) << "Unable to compile UDF file for gpu";
466  return 1;
467  }
468 #endif
469  } else {
470  LOG(FATAL) << "Unable to compile UDF file for cpu";
471  return 1;
472  }
473  } else {
474  LOG(FATAL) << "Unable to create AST file for udf compilation";
475  return 1;
476  }
477 
478  return 0;
479 }
int compileToCpuByteCode(const char *udf_file_name)
#define LOG(tag)
Definition: Logger.h:188
void readCpuCompiledModule()
int parseToAst(const char *file_name)
std::string udf_file_name_
Definition: UDFCompiler.h:79
void readGpuCompiledModule()
int compileForGpu()
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ genCpuIrFilename()

std::string UdfCompiler::genCpuIrFilename ( const char *  udf_file_name)
private

Definition at line 207 of file UDFCompiler.cpp.

Referenced by readCpuCompiledModule().

207  {
208  std::string cpu_file_name(removeFileExtension(udf_fileName));
209 
210  cpu_file_name += "_cpu.bc";
211  return cpu_file_name;
212 }
std::string removeFileExtension(const std::string &path)
+ Here is the caller graph for this function:

◆ genGpuIrFilename()

std::string UdfCompiler::genGpuIrFilename ( const char *  udf_file_name)
private

Definition at line 200 of file UDFCompiler.cpp.

Referenced by readGpuCompiledModule().

200  {
201  std::string gpu_file_name(removeFileExtension(udf_file_name));
202 
203  gpu_file_name += "_gpu.bc";
204  return gpu_file_name;
205 }
std::string removeFileExtension(const std::string &path)
+ Here is the caller graph for this function:

◆ getAstFileName()

const std::string & UdfCompiler::getAstFileName ( ) const

Definition at line 350 of file UDFCompiler.cpp.

Referenced by DBHandler::DBHandler().

350  {
351  return udf_ast_file_name_;
352 }
std::string udf_ast_file_name_
Definition: UDFCompiler.h:80
+ Here is the caller graph for this function:

◆ getFileExt()

std::string UdfCompiler::getFileExt ( std::string &  s)
private

Definition at line 185 of file UDFCompiler.cpp.

185  {
186  size_t i = s.rfind('.', s.length());
187  if (1 != std::string::npos) {
188  return (s.substr(i + 1, s.length() - i));
189  }
190 }

◆ init()

void UdfCompiler::init ( const std::string &  clang_path)
private

Definition at line 354 of file UDFCompiler.cpp.

Referenced by UdfCompiler().

354  {
356 
357  if (clang_path.empty()) {
358  clang_path_.assign(llvm::sys::findProgramByName("clang++").get());
359  if (clang_path_.empty()) {
360  throw std::runtime_error(
361  "Unable to find clang++ to compile user defined functions");
362  }
363  } else {
364  clang_path_.assign(clang_path);
365 
366  if (!boost::filesystem::exists(clang_path)) {
367  throw std::runtime_error("Path provided for udf compiler " + clang_path +
368  " does not exist.");
369  }
370 
371  if (boost::filesystem::is_directory(clang_path)) {
372  throw std::runtime_error("Path provided for udf compiler " + clang_path +
373  " is not to the clang++ executable.");
374  }
375  }
376 }
std::string udf_ast_file_name_
Definition: UDFCompiler.h:80
void replaceExtn(std::string &s, const std::string &new_ext)
std::string clang_path_
Definition: UDFCompiler.h:82
+ Here is the caller graph for this function:

◆ parseToAst()

int UdfCompiler::parseToAst ( const char *  file_name)
private

Definition at line 312 of file UDFCompiler.cpp.

References anonymous_namespace{UDFCompiler.cpp}::convert(), gpu_enabled::copy(), UdfClangDriver::getClangDriver(), UdfClangDriver::the_driver, and ToolingSampleCategory().

Referenced by compileUdf().

312  {
313  UdfClangDriver the_driver(clang_path_);
314  std::string resource_path = the_driver.getClangDriver()->ResourceDir;
315  std::string include_option =
316  std::string("-I") + resource_path + std::string("/include");
317 
318  std::vector<std::string> arg_vector;
319  arg_vector.emplace_back("astparser");
320  arg_vector.emplace_back(file_name);
321  arg_vector.emplace_back("--");
322  arg_vector.emplace_back("-DNO_BOOST");
323  arg_vector.emplace_back(include_option);
324 
325  if (clang_options_.size() > 0) {
326  std::copy(
327  clang_options_.begin(), clang_options_.end(), std::back_inserter(arg_vector));
328  }
329 
330  std::vector<const char*> arg_vec2;
331  std::transform(
332  arg_vector.begin(), arg_vector.end(), std::back_inserter(arg_vec2), convert);
333 
334  int num_args = arg_vec2.size();
335  CommonOptionsParser op(num_args, &arg_vec2[0], ToolingSampleCategory);
336  ClangTool tool(op.getCompilations(), op.getSourcePathList());
337 
338  std::string out_name(file_name);
339  std::string file_ext("ast");
340  replaceExtn(out_name, file_ext);
341 
342  std::error_code out_error_info;
343  llvm::raw_fd_ostream out_file(
344  llvm::StringRef(out_name), out_error_info, llvm::sys::fs::F_None);
345 
346  auto factory = std::make_unique<ToolFactory>(out_file);
347  return tool.run(factory.get());
348 }
std::vector< std::string > clang_options_
Definition: UDFCompiler.h:83
static llvm::cl::OptionCategory ToolingSampleCategory("UDF Tooling")
const char * convert(const std::string &s)
DEVICE auto copy(ARGS &&... args)
Definition: gpu_enabled.h:51
void replaceExtn(std::string &s, const std::string &new_ext)
std::string clang_path_
Definition: UDFCompiler.h:82
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ readCompiledModules()

void UdfCompiler::readCompiledModules ( )
private

Definition at line 414 of file UDFCompiler.cpp.

References readCpuCompiledModule(), and readGpuCompiledModule().

414  {
417 }
void readCpuCompiledModule()
void readGpuCompiledModule()
+ Here is the call graph for this function:

◆ readCpuCompiledModule()

void UdfCompiler::readCpuCompiledModule ( )
private

Definition at line 398 of file UDFCompiler.cpp.

References genCpuIrFilename(), read_udf_cpu_module(), udf_file_name_, and VLOG.

Referenced by compileUdf(), and readCompiledModules().

398  {
399  std::string cpu_ir_file(genCpuIrFilename(udf_file_name_.c_str()));
400 
401  VLOG(1) << "UDFCompiler cpu bc file = " << cpu_ir_file;
402 
403  read_udf_cpu_module(cpu_ir_file);
404 }
std::string genCpuIrFilename(const char *udf_file_name)
void read_udf_cpu_module(const std::string &udf_ir_filename)
std::string udf_file_name_
Definition: UDFCompiler.h:79
#define VLOG(n)
Definition: Logger.h:291
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ readGpuCompiledModule()

void UdfCompiler::readGpuCompiledModule ( )
private

Definition at line 406 of file UDFCompiler.cpp.

References genGpuIrFilename(), read_udf_gpu_module(), udf_file_name_, and VLOG.

Referenced by compileUdf(), and readCompiledModules().

406  {
407  std::string gpu_ir_file(genGpuIrFilename(udf_file_name_.c_str()));
408 
409  VLOG(1) << "UDFCompiler gpu bc file = " << gpu_ir_file;
410 
411  read_udf_gpu_module(gpu_ir_file);
412 }
void read_udf_gpu_module(const std::string &udf_ir_filename)
std::string genGpuIrFilename(const char *udf_file_name)
std::string udf_file_name_
Definition: UDFCompiler.h:79
#define VLOG(n)
Definition: Logger.h:291
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ removeFileExtension()

std::string UdfCompiler::removeFileExtension ( const std::string &  path)
private

Definition at line 172 of file UDFCompiler.cpp.

172  {
173  if (path == "." || path == "..") {
174  return path;
175  }
176 
177  size_t pos = path.find_last_of("\\/.");
178  if (pos != std::string::npos && path[pos] == '.') {
179  return path.substr(0, pos);
180  }
181 
182  return path;
183 }

◆ replaceExtn()

void UdfCompiler::replaceExtn ( std::string &  s,
const std::string &  new_ext 
)
private

Definition at line 192 of file UDFCompiler.cpp.

192  {
193  std::string::size_type i = s.rfind('.', s.length());
194 
195  if (i != std::string::npos) {
196  s.replace(i + 1, getFileExt(s).length(), new_ext);
197  }
198 }
std::string getFileExt(std::string &s)

Member Data Documentation

◆ clang_options_

std::vector<std::string> UdfCompiler::clang_options_
private

Definition at line 83 of file UDFCompiler.h.

◆ clang_path_

std::string UdfCompiler::clang_path_
private

Definition at line 82 of file UDFCompiler.h.

◆ target_arch_

CudaMgr_Namespace::NvidiaDeviceArch UdfCompiler::target_arch_
private

Definition at line 81 of file UDFCompiler.h.

◆ udf_ast_file_name_

std::string UdfCompiler::udf_ast_file_name_
private

Definition at line 80 of file UDFCompiler.h.

◆ udf_file_name_

std::string UdfCompiler::udf_file_name_
private

The documentation for this class was generated from the following files: