[include-fixer] Add a find-all-symbols tool for include-fixer.

Summary:
The find-all-symbols tool generates a yaml symbol database for
include-fixer.

The symbol matcher is originally written by Xiaoyi Liu.

Reviewers: bkramer, djasper

Subscribers: cfe-commits, klimek, ioeric

Differential Revision: http://reviews.llvm.org/D19482

llvm-svn: 267719

clang-tools-extra/include-fixer/CMakeLists.txt

@@ -18,3 +18,4 @@ add_clang_library(clangIncludeFixer
   )
 
 add_subdirectory(tool)
+add_subdirectory(find-all-symbols)

clang-tools-extra/include-fixer/find-all-symbols/CMakeLists.txt

@@ -0,0 +1,18 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+add_clang_library(findAllSymbols
+  FindAllSymbols.cpp
+  SymbolInfo.cpp
+
+  LINK_LIBS
+  clangAST
+  clangASTMatchers
+  clangBasic
+  clangFrontend
+  clangTooling
+  clangToolingCore
+  )
+
+add_subdirectory(tool)

clang-tools-extra/include-fixer/find-all-symbols/FindAllSymbols.cpp

@@ -0,0 +1,193 @@
+//===-- FindAllSymbols.cpp - find all symbols -----------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "FindAllSymbols.h"
+#include "SymbolInfo.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/Type.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/FileSystem.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace find_all_symbols {
+namespace {
+void SetContext(const NamedDecl *ND, SymbolInfo *Symbol) {
+  for (const auto *Context = ND->getDeclContext(); Context;
+       Context = Context->getParent()) {
+    if (llvm::isa<TranslationUnitDecl>(Context) ||
+        llvm::isa<LinkageSpecDecl>(Context))
+      break;
+
+    assert(llvm::isa<NamedDecl>(Context) &&
+           "Expect Context to be a NamedDecl");
+    if (const auto *NSD = dyn_cast<NamespaceDecl>(Context)) {
+      Symbol->Contexts.emplace_back(
+              SymbolInfo::Namespace,
+                  NSD->isAnonymousNamespace() ? "" : NSD->getName().str());
+    } else {
+      const auto *RD = cast<RecordDecl>(Context);
+      Symbol->Contexts.emplace_back(SymbolInfo::Record, RD->getName().str());
+    }
+  }
+}
+
+bool SetCommonInfo(const MatchFinder::MatchResult &Result,
+                   const NamedDecl *ND, SymbolInfo *Symbol) {
+  SetContext(ND, Symbol);
+
+  Symbol->Name = ND->getNameAsString();
+  SourceLocation Loc = Result.SourceManager->getExpansionLoc(ND->getLocation());
+  if (!Loc.isValid()) {
+    llvm::errs() << "Declaration " << ND->getNameAsString() << "("
+                 << ND->getDeclKindName()
+                 << ") has invalid declaration location.";
+    return false;
+  }
+  std::string FilePath = Result.SourceManager->getFilename(Loc).str();
+  if (FilePath.empty())
+    return false;
+
+  Symbol->FilePath = FilePath;
+  Symbol->LineNumber = Result.SourceManager->getExpansionLineNumber(Loc);
+  return true;
+}
+} // namespace
+
+void FindAllSymbols::registerMatchers(MatchFinder *MatchFinder) {
+  // FIXME: Handle specialization.
+  auto IsInSpecialization = hasAncestor(
+      decl(anyOf(cxxRecordDecl(isExplicitTemplateSpecialization()),
+                 functionDecl(isExplicitTemplateSpecialization()))));
+
+  // Matchers for both C and C++.
+  // We only match symbols from header files, i.e. not from main files (see
+  // function's comment for detailed explanation).
+  auto CommonFilter =
+      allOf(unless(isImplicit()), unless(isExpansionInMainFile()));
+
+  auto HasNSOrTUCtxMatcher =
+      hasDeclContext(anyOf(namespaceDecl(), translationUnitDecl()));
+
+  // We need seperate rules for C record types and C++ record types since some
+  // template related matchers are inapplicable on C record declarations.
+  //
+  // Matchers specific to C++ code.
+  // All declarations should be in namespace or translation unit.
+  auto CCMatcher =
+      allOf(HasNSOrTUCtxMatcher, unless(IsInSpecialization),
+            unless(ast_matchers::isTemplateInstantiation()),
+            unless(isInstantiated()), unless(classTemplateSpecializationDecl()),
+            unless(isExplicitTemplateSpecialization()));
+
+  // Matchers specific to code in extern "C" {...}.
+  auto ExternCMatcher = hasDeclContext(linkageSpecDecl());
+
+  // Matchers for variable declarations.
+  //
+  // In most cases, `ParmVarDecl` is filtered out by hasDeclContext(...)
+  // matcher since the declaration context is usually `MethodDecl`. However,
+  // this assumption does not hold for parameters of a function pointer
+  // parameter.
+  // For example, consider a function declaration:
+  //        void Func(void (*)(float), int);
+  // The float parameter of the function pointer has an empty name, and its
+  // declaration context is an anonymous namespace; therefore, it won't be
+  // filtered out by our matchers above.
+  MatchFinder->addMatcher(varDecl(CommonFilter,
+                                  anyOf(ExternCMatcher, CCMatcher),
+                                  unless(parmVarDecl()))
+                              .bind("decl"),
+                          this);
+
+  // Matchers for C-style record declarations in extern "C" {...}.
+  MatchFinder->addMatcher(
+      recordDecl(CommonFilter, ExternCMatcher, isDefinition()).bind("decl"),
+      this);
+
+  // Matchers for C++ record declarations.
+  auto CxxRecordDecl =
+      cxxRecordDecl(CommonFilter, CCMatcher, isDefinition(),
+                    unless(isExplicitTemplateSpecialization()));
+  MatchFinder->addMatcher(CxxRecordDecl.bind("decl"), this);
+
+  // Matchers for function declarations.
+  MatchFinder->addMatcher(
+      functionDecl(CommonFilter, anyOf(ExternCMatcher, CCMatcher)).bind("decl"),
+      this);
+
+  // Matcher for typedef and type alias declarations.
+  //
+  // typedef and type alias can come from C-style headers and C++ heaeders.
+  // For C-style header, `DeclContxet` can be either `TranslationUnitDecl`
+  // or `LinkageSpecDecl`.
+  // For C++ header, `DeclContext ` can be one of `TranslationUnitDecl`,
+  // `NamespaceDecl`.
+  // With the following context matcher, we can match `typedefNameDecl` from
+  // both C-style header and C++ header (except for those in classes).
+  // "cc_matchers" are not included since template-related matchers are not
+  // applicable on `TypedefNameDecl`.
+  MatchFinder->addMatcher(
+      typedefNameDecl(CommonFilter, anyOf(HasNSOrTUCtxMatcher,
+                                          hasDeclContext(linkageSpecDecl())))
+          .bind("decl"),
+      this);
+}
+
+void FindAllSymbols::run(const MatchFinder::MatchResult &Result) {
+  // Ignore Results in failing TUs.
+  if (Result.Context->getDiagnostics().hasErrorOccurred()) {
+    return;
+  }
+
+  const NamedDecl *ND = Result.Nodes.getNodeAs<NamedDecl>("decl");
+  assert(ND && "Matched declaration must be a NamedDecl!");
+  const SourceManager *SM = Result.SourceManager;
+
+  SymbolInfo Symbol;
+  if (!SetCommonInfo(Result, ND, &Symbol))
+    return;
+
+  if (const auto *VD = llvm::dyn_cast<VarDecl>(ND)) {
+    Symbol.Type = SymbolInfo::Variable;
+    SymbolInfo::VariableInfo VI;
+    VI.Type = VD->getType().getAsString();
+    Symbol.VariableInfos = VI;
+  } else if (const auto *FD = llvm::dyn_cast<FunctionDecl>(ND)) {
+    Symbol.Type = SymbolInfo::Function;
+    SymbolInfo::FunctionInfo FI;
+    FI.ReturnType = FD->getReturnType().getAsString();
+    for (const auto *Param : FD->params())
+      FI.ParameterTypes.push_back(Param->getType().getAsString());
+    Symbol.FunctionInfos = FI;
+  } else if (const auto *TD = llvm::dyn_cast<TypedefNameDecl>(ND)) {
+    Symbol.Type = SymbolInfo::TypedefName;
+    SymbolInfo::TypedefNameInfo TI;
+    TI.UnderlyingType = TD->getUnderlyingType().getAsString();
+    Symbol.TypedefNameInfos = TI;
+  } else {
+    assert(
+        llvm::isa<RecordDecl>(ND) &&
+        "Matched decl must be one of VarDecl, FunctionDecl, and RecordDecl!");
+    // C-style record decl can have empty name, e.g "struct { ... } var;".
+    if (ND->getName().empty())
+      return;
+    Symbol.Type = SymbolInfo::Class;
+  }
+
+  const FileEntry *FE = SM->getFileEntryForID(SM->getMainFileID());
+  Reporter->reportResult(FE->getName(), Symbol);
+}
+
+} // namespace find_all_symbols
+} // namespace clang

clang-tools-extra/include-fixer/find-all-symbols/FindAllSymbols.h

@@ -0,0 +1,56 @@
+//===-- FindAllSymbols.h - find all symbols----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H
+#define LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H
+
+#include "SymbolInfo.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include <string>
+
+namespace clang {
+namespace find_all_symbols {
+
+/// \brief FindAllSymbols collects all classes, free standing functions and
+/// global variables with some extra information such as the path of the header
+/// file, the namespaces they are contained in, the type of variables and the
+/// parameter types of functions.
+///
+/// NOTE:
+///   - Symbols declared in main files are not collected since they can not be
+///   included.
+///   - Member functions are not collected because accessing them must go
+///   through the class. #include fixer only needs the class name to find
+///   headers.
+///
+class FindAllSymbols : public clang::ast_matchers::MatchFinder::MatchCallback {
+public:
+  class ResultReporter {
+  public:
+    virtual ~ResultReporter() = default;
+
+    virtual void reportResult(llvm::StringRef FileName,
+                              const SymbolInfo &Symbol) = 0;
+  };
+
+  explicit FindAllSymbols(ResultReporter *Reporter) : Reporter(Reporter) {}
+
+  void registerMatchers(clang::ast_matchers::MatchFinder *MatchFinder);
+
+  void
+  run(const clang::ast_matchers::MatchFinder::MatchResult &result) override;
+
+private:
+  ResultReporter *const Reporter;
+};
+
+} // namespace find_all_symbols
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H

clang-tools-extra/include-fixer/find-all-symbols/SymbolInfo.cpp

@@ -0,0 +1,121 @@
+//===-- SymbolInfo.cpp ----------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+
+using llvm::yaml::MappingTraits;
+using llvm::yaml::IO;
+using llvm::yaml::Input;
+using ContextType = clang::find_all_symbols::SymbolInfo::ContextType;
+using clang::find_all_symbols::SymbolInfo;
+using SymbolKind = clang::find_all_symbols::SymbolInfo::SymbolKind;
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(SymbolInfo)
+LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(std::string)
+LLVM_YAML_IS_SEQUENCE_VECTOR(SymbolInfo::Context)
+
+namespace llvm {
+namespace yaml {
+template <> struct MappingTraits<SymbolInfo> {
+  static void mapping(IO &io, SymbolInfo &Symbol) {
+    io.mapRequired("Name", Symbol.Name);
+    io.mapRequired("Contexts", Symbol.Contexts);
+    io.mapRequired("FilePath", Symbol.FilePath);
+    io.mapRequired("LineNumber", Symbol.LineNumber);
+    io.mapRequired("Type", Symbol.Type);
+    io.mapOptional("Variable", Symbol.VariableInfos);
+    io.mapOptional("Function", Symbol.FunctionInfos);
+    io.mapOptional("TypedefName", Symbol.TypedefNameInfos);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<ContextType> {
+  static void enumeration(IO &io, ContextType &value) {
+    io.enumCase(value, "Record", ContextType::Record);
+    io.enumCase(value, "Namespace", ContextType::Namespace);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolKind> {
+  static void enumeration(IO &io, SymbolKind &value) {
+    io.enumCase(value, "Variable", SymbolKind::Variable);
+    io.enumCase(value, "Function", SymbolKind::Function);
+    io.enumCase(value, "Class", SymbolKind::Class);
+    io.enumCase(value, "TypedefName", SymbolKind::TypedefName);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo::Context> {
+  static void mapping(IO &io, SymbolInfo::Context &Context) {
+    io.mapRequired("ContextType", Context.first);
+    io.mapRequired("ContextName", Context.second);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo::FunctionInfo> {
+  static void mapping(IO &io, SymbolInfo::FunctionInfo &Value) {
+    io.mapRequired("ReturnType", Value.ReturnType);
+    io.mapRequired("ParameterTypes", Value.ParameterTypes);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo::VariableInfo> {
+  static void mapping(IO &io, SymbolInfo::VariableInfo &Value) {
+    io.mapRequired("VariableType", Value.Type);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo::TypedefNameInfo> {
+  static void mapping(IO &io, SymbolInfo::TypedefNameInfo &Value) {
+    io.mapRequired("TypedefNameType", Value.UnderlyingType);
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+namespace clang {
+namespace find_all_symbols {
+
+bool SymbolInfo::operator==(const SymbolInfo &Symbol) const {
+  return Name == Symbol.Name && FilePath == Symbol.FilePath &&
+         LineNumber == Symbol.LineNumber && Contexts == Symbol.Contexts;
+}
+
+bool SymbolInfo::operator<(const SymbolInfo &Symbol) const {
+  return std::tie(Name, FilePath, LineNumber) <
+         std::tie(Symbol.Name, Symbol.FilePath, Symbol.LineNumber);
+}
+
+bool WriteSymboInfosToFile(llvm::StringRef FilePath,
+                           const std::set<SymbolInfo> &Symbols) {
+  int FD = 0;
+  if (llvm::sys::fs::openFileForWrite(FilePath, FD, llvm::sys::fs::F_None))
+    return false;
+  llvm::raw_fd_ostream OS(FD, true);
+  llvm::yaml::Output yout(OS);
+  for (auto Symbol : Symbols)
+    yout << Symbol;
+  OS.close();
+  return true;
+}
+
+std::vector<SymbolInfo> ReadSymbolInfosFromYAML(llvm::StringRef Yaml) {
+  std::vector<SymbolInfo> Symbols;
+  llvm::yaml::Input yin(Yaml);
+  yin >> Symbols;
+  return Symbols;
+}
+
+} // namespace find_all_symbols
+} // namespace clang