OmniSciDB  72180abbfe
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
UDFCompiler.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "UDFCompiler.h"
18 
19 #include <clang/AST/AST.h>
20 #include <clang/AST/ASTConsumer.h>
21 #include <clang/AST/RecursiveASTVisitor.h>
22 #include <clang/Driver/Compilation.h>
23 #include <clang/Driver/Driver.h>
24 #include <clang/Frontend/CompilerInstance.h>
25 #include <clang/Frontend/FrontendActions.h>
26 #include <clang/Frontend/TextDiagnosticPrinter.h>
27 #include <clang/Parse/ParseAST.h>
28 #include <clang/Tooling/CommonOptionsParser.h>
29 #include <clang/Tooling/Tooling.h>
30 #include <llvm/Support/Program.h>
31 #include <llvm/Support/raw_ostream.h>
32 #include <boost/process/search_path.hpp>
33 #include <iterator>
34 #include <memory>
35 
36 #include "Execute.h"
37 #include "Shared/Logger.h"
38 
39 using namespace clang;
40 using namespace clang::tooling;
41 
42 static llvm::cl::OptionCategory ToolingSampleCategory("UDF Tooling");
43 
44 namespace {
45 
46 // By implementing RecursiveASTVisitor, we can specify which AST nodes
47 // we're interested in by overriding relevant methods.
48 
49 class FunctionDeclVisitor : public RecursiveASTVisitor<FunctionDeclVisitor> {
50  public:
51  FunctionDeclVisitor(llvm::raw_fd_ostream& ast_file,
52  SourceManager& s_manager,
53  ASTContext& context)
54  : ast_file_(ast_file), source_manager_(s_manager), context_(context) {
55  source_manager_.getDiagnostics().setShowColors(false);
56  }
57 
58  bool VisitFunctionDecl(FunctionDecl* f) {
59  // Only function definitions (with bodies), not declarations.
60  if (f->hasBody()) {
61  if (getMainFileName() == getFuncDeclFileName(f)) {
62  auto printing_policy = context_.getPrintingPolicy();
63  printing_policy.FullyQualifiedName = 1;
64  printing_policy.UseVoidForZeroParams = 1;
65  printing_policy.PolishForDeclaration = 1;
66  printing_policy.TerseOutput = 1;
67  f->print(ast_file_, printing_policy);
68  ast_file_ << "\n";
69  }
70  }
71 
72  return true;
73  }
74 
75  private:
76  std::string getMainFileName() const {
77  auto f_entry = source_manager_.getFileEntryForID(source_manager_.getMainFileID());
78  return f_entry->getName().str();
79  }
80 
81  std::string getFuncDeclFileName(FunctionDecl* f) const {
82  SourceLocation spell_loc = source_manager_.getSpellingLoc(f->getLocation());
83  PresumedLoc p_loc = source_manager_.getPresumedLoc(spell_loc);
84 
85  return std::string(p_loc.getFilename());
86  }
87 
88  private:
89  llvm::raw_fd_ostream& ast_file_;
90  SourceManager& source_manager_;
91  ASTContext& context_;
92 };
93 
94 // Implementation of the ASTConsumer interface for reading an AST produced
95 // by the Clang parser.
96 class DeclASTConsumer : public ASTConsumer {
97  public:
98  DeclASTConsumer(llvm::raw_fd_ostream& ast_file,
99  SourceManager& s_manager,
100  ASTContext& context)
101  : visitor_(ast_file, s_manager, context) {}
102 
103  // Override the method that gets called for each parsed top-level
104  // declaration.
105  bool HandleTopLevelDecl(DeclGroupRef decl_reference) override {
106  for (DeclGroupRef::iterator b = decl_reference.begin(), e = decl_reference.end();
107  b != e;
108  ++b) {
109  // Traverse the declaration using our AST visitor.
110  visitor_.TraverseDecl(*b);
111  }
112  return true;
113  }
114 
115  private:
117 };
118 
119 // For each source file provided to the tool, a new FrontendAction is created.
120 class HandleDeclAction : public ASTFrontendAction {
121  public:
122  HandleDeclAction(llvm::raw_fd_ostream& ast_file) : ast_file_(ast_file) {}
123 
124  ~HandleDeclAction() override {}
125 
126  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance& instance,
127  StringRef file) override {
128  return std::make_unique<DeclASTConsumer>(
129  ast_file_, instance.getSourceManager(), instance.getASTContext());
130  }
131 
132  private:
133  llvm::raw_fd_ostream& ast_file_;
134 };
135 
136 class ToolFactory : public FrontendActionFactory {
137  public:
138 #if LLVM_VERSION_MAJOR >= 10
139  using FrontendActionPtr = std::unique_ptr<clang::FrontendAction>;
140 #define CREATE_FRONTEND_ACTION(ast_file_) std::make_unique<HandleDeclAction>(ast_file_)
141 #else
142  using FrontendActionPtr = clang::FrontendAction*;
143 #define CREATE_FRONTEND_ACTION(ast_file_) new HandleDeclAction(ast_file_)
144 #endif
145 
146  ToolFactory(llvm::raw_fd_ostream& ast_file) : ast_file_(ast_file) {}
147 
148  FrontendActionPtr create() override { return CREATE_FRONTEND_ACTION(ast_file_); }
149 
150  private:
151  llvm::raw_fd_ostream& ast_file_;
152 };
153 
154 const char* convert(const std::string& s) {
155  return s.c_str();
156 }
157 } // namespace
158 
159 UdfClangDriver::UdfClangDriver(const std::string& clang_path)
160  : diag_options(new DiagnosticOptions())
161  , diag_client(new TextDiagnosticPrinter(llvm::errs(), diag_options.get()))
162  , diag_id(new clang::DiagnosticIDs())
163  , diags(diag_id, diag_options.get(), diag_client)
164  , diag_client_owner(diags.takeClient())
165  , the_driver(clang_path.c_str(), llvm::sys::getDefaultTargetTriple(), diags) {}
166 
167 std::string UdfCompiler::removeFileExtension(const std::string& path) {
168  if (path == "." || path == "..") {
169  return path;
170  }
171 
172  size_t pos = path.find_last_of("\\/.");
173  if (pos != std::string::npos && path[pos] == '.') {
174  return path.substr(0, pos);
175  }
176 
177  return path;
178 }
179 
180 std::string UdfCompiler::getFileExt(std::string& s) {
181  size_t i = s.rfind('.', s.length());
182  if (1 != std::string::npos) {
183  return (s.substr(i + 1, s.length() - i));
184  }
185 }
186 
187 void UdfCompiler::replaceExtn(std::string& s, const std::string& new_ext) {
188  std::string::size_type i = s.rfind('.', s.length());
189 
190  if (i != std::string::npos) {
191  s.replace(i + 1, getFileExt(s).length(), new_ext);
192  }
193 }
194 
195 std::string UdfCompiler::genGpuIrFilename(const char* udf_file_name) {
196  std::string gpu_file_name(removeFileExtension(udf_file_name));
197 
198  gpu_file_name += "_gpu.bc";
199  return gpu_file_name;
200 }
201 
202 std::string UdfCompiler::genCpuIrFilename(const char* udf_fileName) {
203  std::string cpu_file_name(removeFileExtension(udf_fileName));
204 
205  cpu_file_name += "_cpu.bc";
206  return cpu_file_name;
207 }
208 
209 int UdfCompiler::compileFromCommandLine(const std::vector<std::string>& command_line) {
210  UdfClangDriver compiler_driver(clang_path_);
211  auto the_driver(compiler_driver.getClangDriver());
212 
213  std::vector<const char*> clang_command_opts;
214  clang_command_opts.reserve(command_line.size() + clang_options_.size());
215  // add required options first
216  std::transform(std::begin(command_line),
217  std::end(command_line),
218  std::back_inserter(clang_command_opts),
219  [&](const std::string& str) { return str.c_str(); });
220 
221  // If there were additional clang options passed to the system, append them here
222  if (!clang_options_.empty()) {
223  std::transform(std::begin(clang_options_),
224  std::end(clang_options_),
225  std::back_inserter(clang_command_opts),
226  [&](const std::string& str) { return str.c_str(); });
227  }
228 
229  the_driver->CCPrintOptions = 0;
230  std::unique_ptr<driver::Compilation> compilation(
231  the_driver->BuildCompilation(clang_command_opts));
232 
233  if (!compilation) {
234  LOG(FATAL) << "failed to build compilation object!\n";
235  }
236 
237  llvm::SmallVector<std::pair<int, const driver::Command*>, 10> failing_commands;
238  int res = the_driver->ExecuteCompilation(*compilation, failing_commands);
239 
240  if (res < 0) {
241  for (const std::pair<int, const driver::Command*>& p : failing_commands) {
242  if (p.first) {
243  the_driver->generateCompilationDiagnostics(*compilation, *p.second);
244  }
245  }
246  }
247 
248  return res;
249 }
250 
251 int UdfCompiler::compileToGpuByteCode(const char* udf_file_name, bool cpu_mode) {
252  std::string gpu_out_filename(genGpuIrFilename(udf_file_name));
253 
254  std::vector<std::string> command_line{
255  clang_path_, "-c", "-O2", "-emit-llvm", "-o", gpu_out_filename, "-std=c++14"};
256 
257  // If we are not compiling for cpu mode, then target the gpu
258  // Otherwise assume we can generic ir that will
259  // be translated to gpu code during target code generation
260  if (!cpu_mode) {
261  command_line.emplace_back("--cuda-gpu-arch=" +
263  command_line.emplace_back("--cuda-device-only");
264  command_line.emplace_back("-xcuda");
265  }
266 
267  command_line.emplace_back(udf_file_name);
268 
269  return compileFromCommandLine(command_line);
270 }
271 
272 int UdfCompiler::compileToCpuByteCode(const char* udf_file_name) {
273  std::string cpu_out_filename(genCpuIrFilename(udf_file_name));
274 
275  std::vector<std::string> command_line{clang_path_,
276  "-c",
277  "-O2",
278  "-emit-llvm",
279  "-o",
280  cpu_out_filename,
281  "-std=c++14",
282  udf_file_name};
283 
284  return compileFromCommandLine(command_line);
285 }
286 
287 int UdfCompiler::parseToAst(const char* file_name) {
288  UdfClangDriver the_driver(clang_path_);
289  std::string resource_path = the_driver.getClangDriver()->ResourceDir;
290  std::string include_option =
291  std::string("-I") + resource_path + std::string("/include");
292 
293  std::vector<std::string> arg_vector;
294  arg_vector.emplace_back("astparser");
295  arg_vector.emplace_back(file_name);
296  arg_vector.emplace_back("--");
297  arg_vector.emplace_back(include_option);
298 
299  if (clang_options_.size() > 0) {
300  std::copy(
301  clang_options_.begin(), clang_options_.end(), std::back_inserter(arg_vector));
302  }
303 
304  std::vector<const char*> arg_vec2;
305  std::transform(
306  arg_vector.begin(), arg_vector.end(), std::back_inserter(arg_vec2), convert);
307 
308  int num_args = arg_vec2.size();
309  CommonOptionsParser op(num_args, &arg_vec2[0], ToolingSampleCategory);
310  ClangTool tool(op.getCompilations(), op.getSourcePathList());
311 
312  std::string out_name(file_name);
313  std::string file_ext("ast");
314  replaceExtn(out_name, file_ext);
315 
316  std::error_code out_error_info;
317  llvm::raw_fd_ostream out_file(
318  llvm::StringRef(out_name), out_error_info, llvm::sys::fs::F_None);
319 
320  auto factory = std::make_unique<ToolFactory>(out_file);
321  return tool.run(factory.get());
322 }
323 
324 const std::string& UdfCompiler::getAstFileName() const {
325  return udf_ast_file_name_;
326 }
327 
328 void UdfCompiler::init(const std::string& clang_path) {
330 
331  if (clang_path.empty()) {
332  clang_path_.assign(llvm::sys::findProgramByName("clang++").get());
333  if (clang_path_.empty()) {
334  throw std::runtime_error(
335  "Unable to find clang++ to compile user defined functions");
336  }
337  } else {
338  clang_path_.assign(clang_path);
339 
340  if (!boost::filesystem::exists(clang_path)) {
341  throw std::runtime_error("Path provided for udf compiler " + clang_path +
342  " does not exist.");
343  }
344 
345  if (boost::filesystem::is_directory(clang_path)) {
346  throw std::runtime_error("Path provided for udf compiler " + clang_path +
347  " is not to the clang++ executable.");
348  }
349  }
350 }
351 
352 UdfCompiler::UdfCompiler(const std::string& file_name,
354  const std::string& clang_path)
355  : udf_file_name_(file_name)
356  , udf_ast_file_name_(file_name)
357  , target_arch_(target_arch) {
358  init(clang_path);
359 }
360 
361 UdfCompiler::UdfCompiler(const std::string& file_name,
363  const std::string& clang_path,
364  const std::vector<std::string> clang_options)
365  : udf_file_name_(file_name)
366  , udf_ast_file_name_(file_name)
367  , target_arch_(target_arch)
368  , clang_options_(clang_options) {
369  init(clang_path);
370 }
371 
373  std::string cpu_ir_file(genCpuIrFilename(udf_file_name_.c_str()));
374 
375  VLOG(1) << "UDFCompiler cpu bc file = " << cpu_ir_file;
376 
377  read_udf_cpu_module(cpu_ir_file);
378 }
379 
381  std::string gpu_ir_file(genGpuIrFilename(udf_file_name_.c_str()));
382 
383  VLOG(1) << "UDFCompiler gpu bc file = " << gpu_ir_file;
384 
385  read_udf_gpu_module(gpu_ir_file);
386 }
387 
391 }
392 
394  int gpu_compile_result = 1;
395 
396  gpu_compile_result = compileToGpuByteCode(udf_file_name_.c_str(), false);
397 
398  // If gpu compilation fails but cpu compilation has succeeded, try compiling
399  // for the cpu with the assumption the user does not have the CUDA toolkit
400  // installed
401  if (gpu_compile_result != 0) {
402  gpu_compile_result = compileToGpuByteCode(udf_file_name_.c_str(), true);
403  }
404 
405  return gpu_compile_result;
406 }
407 
409  LOG(INFO) << "UDFCompiler filename to compile: " << udf_file_name_;
410  if (!boost::filesystem::exists(udf_file_name_)) {
411  LOG(FATAL) << "User defined function file " << udf_file_name_ << " does not exist.";
412  return 1;
413  }
414 
415  auto ast_result = parseToAst(udf_file_name_.c_str());
416 
417  if (ast_result == 0) {
418  // Compile udf file to generate cpu and gpu bytecode files
419 
420  int cpu_compile_result = compileToCpuByteCode(udf_file_name_.c_str());
421 #ifdef HAVE_CUDA
422  int gpu_compile_result = 1;
423 #endif
424 
425  if (cpu_compile_result == 0) {
427 #ifdef HAVE_CUDA
428  gpu_compile_result = compileForGpu();
429 
430  if (gpu_compile_result == 0) {
432  } else {
433  LOG(FATAL) << "Unable to compile UDF file for gpu";
434  return 1;
435  }
436 #endif
437  } else {
438  LOG(FATAL) << "Unable to compile UDF file for cpu";
439  return 1;
440  }
441  } else {
442  LOG(FATAL) << "Unable to create AST file for udf compilation";
443  return 1;
444  }
445 
446  return 0;
447 }
ToolFactory(llvm::raw_fd_ostream &ast_file)
clang::driver::Driver * getClangDriver()
Definition: UDFCompiler.h:39
std::string genCpuIrFilename(const char *udf_file_name)
int compileToCpuByteCode(const char *udf_file_name)
void readCompiledModules()
std::vector< std::string > clang_options_
Definition: UDFCompiler.h:83
UdfCompiler(const std::string &udf_file_name, CudaMgr_Namespace::NvidiaDeviceArch target_arch, const std::string &clang_path="")
const std::string & getAstFileName() const
#define LOG(tag)
Definition: Logger.h:188
void readCpuCompiledModule()
void read_udf_cpu_module(const std::string &udf_ir_filename)
static llvm::cl::OptionCategory ToolingSampleCategory("UDF Tooling")
void read_udf_gpu_module(const std::string &udf_ir_filename)
External interface for parsing AST and bitcode files.
std::string genGpuIrFilename(const char *udf_file_name)
const char * convert(const std::string &s)
std::string removeFileExtension(const std::string &path)
DeclASTConsumer(llvm::raw_fd_ostream &ast_file, SourceManager &s_manager, ASTContext &context)
Definition: UDFCompiler.cpp:98
int compileToGpuByteCode(const char *udf_file_name, bool cpu_mode)
std::unique_ptr< ASTConsumer > CreateASTConsumer(CompilerInstance &instance, StringRef file) override
std::string udf_ast_file_name_
Definition: UDFCompiler.h:80
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t uint32_t const int64_t int32_t * error_code
UdfClangDriver(const std::string &)
int parseToAst(const char *file_name)
bool HandleTopLevelDecl(DeclGroupRef decl_reference) override
CudaMgr_Namespace::NvidiaDeviceArch target_arch_
Definition: UDFCompiler.h:81
static std::string deviceArchToSM(const NvidiaDeviceArch arch)
Definition: CudaMgr.h:157
void init(const std::string &clang_path)
std::string udf_file_name_
Definition: UDFCompiler.h:79
int compileUdf()
void replaceExtn(std::string &s, const std::string &new_ext)
std::string getFuncDeclFileName(FunctionDecl *f) const
Definition: UDFCompiler.cpp:81
void readGpuCompiledModule()
std::string getFileExt(std::string &s)
int compileForGpu()
FunctionDeclVisitor(llvm::raw_fd_ostream &ast_file, SourceManager &s_manager, ASTContext &context)
Definition: UDFCompiler.cpp:51
int compileFromCommandLine(const std::vector< std::string > &command_line)
#define VLOG(n)
Definition: Logger.h:291
std::string clang_path_
Definition: UDFCompiler.h:82
#define CREATE_FRONTEND_ACTION(ast_file_)