Skip to content

Commit

Permalink
[clangd] Using index for GoToDefinition.
Browse files Browse the repository at this point in the history 
Summary:
This patch adds index support for GoToDefinition -- when we don't get the
definition from local AST, we query our index (Static&Dynamic) index to
get it.

Since we currently collect top-level symbol in the index, it doesn't support all
cases (e.g. class members), we will extend the index to include more symbols in
the future.

Reviewers: sammccall

Subscribers: klimek, ilya-biryukov, jkorous-apple, ioeric, MaskRay, cfe-commits

Differential Revision: https://reviews.llvm.org/D45717

llvm-svn: 331189
  • Loading branch information
@hokein
hokein committed Apr 30, 2018
1 parent 0a732da commit 88cfb66
Show file tree
Hide file tree
Showing 6 changed files with 275 additions and 85 deletions.
6 changes: 3 additions & 3 deletions clang-tools-extra/clangd/ClangdServer.cpp
View file
Expand Up @@ -355,11 +355,11 @@ void ClangdServer::dumpAST(PathRef File,
void ClangdServer::findDefinitions(PathRef File, Position Pos,
Callback<std::vector<Location>> CB) {
auto FS = FSProvider.getFileSystem();
auto Action = [Pos, FS](Callback<std::vector<Location>> CB,
llvm::Expected<InputsAndAST> InpAST) {
auto Action = [Pos, FS, this](Callback<std::vector<Location>> CB,
llvm::Expected<InputsAndAST> InpAST) {
if (!InpAST)
return CB(InpAST.takeError());
CB(clangd::findDefinitions(InpAST->AST, Pos));
CB(clangd::findDefinitions(InpAST->AST, Pos, this->FileIdx.get()));
};

WorkScheduler.runWithAST("Definitions", File, Bind(Action, std::move(CB)));
Expand Down
232 changes: 172 additions & 60 deletions clang-tools-extra/clangd/XRefs.cpp
View file
Expand Up @@ -14,6 +14,7 @@
#include "clang/AST/DeclTemplate.h"
#include "clang/Index/IndexDataConsumer.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Index/USRGeneration.h"
#include "llvm/Support/Path.h"
namespace clang {
namespace clangd {
Expand All @@ -34,6 +35,33 @@ const Decl *GetDefinition(const Decl *D) {
return nullptr;
}

// Convert a SymbolLocation to LSP's Location.
// HintPath is used to resolve the path of URI.
// FIXME: figure out a good home for it, and share the implementation with
// FindSymbols.
llvm::Optional<Location> ToLSPLocation(const SymbolLocation &Loc,
llvm::StringRef HintPath) {
if (!Loc)
return llvm::None;
auto Uri = URI::parse(Loc.FileURI);
if (!Uri) {
log("Could not parse URI: " + Loc.FileURI);
return llvm::None;
}
auto Path = URI::resolve(*Uri, HintPath);
if (!Path) {
log("Could not resolve URI: " + Loc.FileURI);
return llvm::None;
}
Location LSPLoc;
LSPLoc.uri = URIForFile(*Path);
LSPLoc.range.start.line = Loc.Start.Line;
LSPLoc.range.start.character = Loc.Start.Column;
LSPLoc.range.end.line = Loc.End.Line;
LSPLoc.range.end.character = Loc.End.Column;
return LSPLoc;
}

struct MacroDecl {
StringRef Name;
const MacroInfo *Info;
Expand Down Expand Up @@ -128,6 +156,38 @@ class DeclarationAndMacrosFinder : public index::IndexDataConsumer {
}
};

struct IdentifiedSymbol {
std::vector<const Decl *> Decls;
std::vector<MacroDecl> Macros;
};

IdentifiedSymbol getSymbolAtPosition(ParsedAST &AST, SourceLocation Pos) {
auto DeclMacrosFinder = DeclarationAndMacrosFinder(
llvm::errs(), Pos, AST.getASTContext(), AST.getPreprocessor());
index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
IndexOpts.IndexFunctionLocals = true;
indexTopLevelDecls(AST.getASTContext(), AST.getTopLevelDecls(),
DeclMacrosFinder, IndexOpts);

return {DeclMacrosFinder.takeDecls(), DeclMacrosFinder.takeMacroInfos()};
}

llvm::Optional<std::string>
getAbsoluteFilePath(const FileEntry *F, const SourceManager &SourceMgr) {
SmallString<64> FilePath = F->tryGetRealPathName();
if (FilePath.empty())
FilePath = F->getName();
if (!llvm::sys::path::is_absolute(FilePath)) {
if (!SourceMgr.getFileManager().makeAbsolutePath(FilePath)) {
log("Could not turn relative path to absolute: " + FilePath);
return llvm::None;
}
}
return FilePath.str().str();
}

llvm::Optional<Location>
makeLocation(ParsedAST &AST, const SourceRange &ValSourceRange) {
const SourceManager &SourceMgr = AST.getASTContext().getSourceManager();
Expand All @@ -145,24 +205,29 @@ makeLocation(ParsedAST &AST, const SourceRange &ValSourceRange) {
Range R = {Begin, End};
Location L;

SmallString<64> FilePath = F->tryGetRealPathName();
if (FilePath.empty())
FilePath = F->getName();
if (!llvm::sys::path::is_absolute(FilePath)) {
if (!SourceMgr.getFileManager().makeAbsolutePath(FilePath)) {
log("Could not turn relative path to absolute: " + FilePath);
return llvm::None;
}
auto FilePath = getAbsoluteFilePath(F, SourceMgr);
if (!FilePath) {
log("failed to get path!");
return llvm::None;
}

L.uri = URIForFile(FilePath.str());
L.uri = URIForFile(*FilePath);
L.range = R;
return L;
}

// Get the symbol ID for a declaration, if possible.
llvm::Optional<SymbolID> getSymbolID(const Decl *D) {
llvm::SmallString<128> USR;
if (index::generateUSRForDecl(D, USR)) {
return None;
}
return SymbolID(USR);
}

} // namespace

std::vector<Location> findDefinitions(ParsedAST &AST, Position Pos) {
std::vector<Location> findDefinitions(ParsedAST &AST, Position Pos,
const SymbolIndex *Index) {
const SourceManager &SourceMgr = AST.getASTContext().getSourceManager();
SourceLocation SourceLocationBeg =
getBeginningOfIdentifier(AST, Pos, SourceMgr.getMainFileID());
Expand All @@ -179,32 +244,97 @@ std::vector<Location> findDefinitions(ParsedAST &AST, Position Pos) {
if (!Result.empty())
return Result;

DeclarationAndMacrosFinder DeclMacrosFinder(llvm::errs(), SourceLocationBeg,
AST.getASTContext(),
AST.getPreprocessor());
index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
IndexOpts.IndexFunctionLocals = true;

indexTopLevelDecls(AST.getASTContext(), AST.getTopLevelDecls(),
DeclMacrosFinder, IndexOpts);

std::vector<const Decl *> Decls = DeclMacrosFinder.takeDecls();
std::vector<MacroDecl> MacroInfos = DeclMacrosFinder.takeMacroInfos();
// Identified symbols at a specific position.
auto Symbols = getSymbolAtPosition(AST, SourceLocationBeg);

for (auto D : Decls) {
auto Loc = findNameLoc(D);
for (auto Item : Symbols.Macros) {
auto Loc = Item.Info->getDefinitionLoc();
auto L = makeLocation(AST, SourceRange(Loc, Loc));
if (L)
Result.push_back(*L);
}

for (auto Item : MacroInfos) {
auto Loc = Item.Info->getDefinitionLoc();
// Declaration and definition are different terms in C-family languages, and
// LSP only defines the "GoToDefinition" specification, so we try to perform
// the "most sensible" GoTo operation:
//
// - We use the location from AST and index (if available) to provide the
// final results. When there are duplicate results, we prefer AST over
// index because AST is more up-to-date.
//
// - For each symbol, we will return a location of the canonical declaration
// (e.g. function declaration in header), and a location of definition if
// they are available.
//
// So the work flow:
//
// 1. Identify the symbols being search for by traversing the AST.
// 2. Populate one of the locations with the AST location.
// 3. Use the AST information to query the index, and populate the index
// location (if available).
// 4. Return all populated locations for all symbols, definition first (
// which we think is the users wants most often).
struct CandidateLocation {
llvm::Optional<Location> Def;
llvm::Optional<Location> Decl;
};
llvm::DenseMap<SymbolID, CandidateLocation> ResultCandidates;

// Emit all symbol locations (declaration or definition) from AST.
for (const auto *D : Symbols.Decls) {
// Fake key for symbols don't have USR (no SymbolID).
// Ideally, there should be a USR for each identified symbols. Symbols
// without USR are rare and unimportant cases, we use the a fake holder to
// minimize the invasiveness of these cases.
SymbolID Key("");
if (auto ID = getSymbolID(D))
Key = *ID;

auto &Candidate = ResultCandidates[Key];
auto Loc = findNameLoc(D);
auto L = makeLocation(AST, SourceRange(Loc, Loc));
if (L)
Result.push_back(*L);
// The declaration in the identified symbols is a definition if possible
// otherwise it is declaration.
bool IsDef = GetDefinition(D) == D;
// Populate one of the slots with location for the AST.
if (!IsDef)
Candidate.Decl = L;
else
Candidate.Def = L;
}

if (Index) {
LookupRequest QueryRequest;
// Build request for index query, using SymbolID.
for (auto It : ResultCandidates)
QueryRequest.IDs.insert(It.first);
std::string HintPath;
const FileEntry *FE =
SourceMgr.getFileEntryForID(SourceMgr.getMainFileID());
if (auto Path = getAbsoluteFilePath(FE, SourceMgr))
HintPath = *Path;
// Query the index and populate the empty slot.
Index->lookup(
QueryRequest, [&HintPath, &ResultCandidates](const Symbol &Sym) {
auto It = ResultCandidates.find(Sym.ID);
assert(It != ResultCandidates.end());
auto &Value = It->second;

if (!Value.Def)
Value.Def = ToLSPLocation(Sym.Definition, HintPath);
if (!Value.Decl)
Value.Decl = ToLSPLocation(Sym.CanonicalDeclaration, HintPath);
});
}

// Populate the results, definition first.
for (auto It : ResultCandidates) {
const auto &Candidate = It.second;
if (Candidate.Def)
Result.push_back(*Candidate.Def);
if (Candidate.Decl &&
Candidate.Decl != Candidate.Def) // Decl and Def might be the same
Result.push_back(*Candidate.Decl);
}

return Result;
Expand Down Expand Up @@ -280,23 +410,16 @@ std::vector<DocumentHighlight> findDocumentHighlights(ParsedAST &AST,
SourceLocation SourceLocationBeg =
getBeginningOfIdentifier(AST, Pos, SourceMgr.getMainFileID());

DeclarationAndMacrosFinder DeclMacrosFinder(llvm::errs(), SourceLocationBeg,
AST.getASTContext(),
AST.getPreprocessor());
index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
IndexOpts.IndexFunctionLocals = true;

// Macro occurences are not currently handled.
indexTopLevelDecls(AST.getASTContext(), AST.getTopLevelDecls(),
DeclMacrosFinder, IndexOpts);

std::vector<const Decl *> SelectedDecls = DeclMacrosFinder.takeDecls();
auto Symbols = getSymbolAtPosition(AST, SourceLocationBeg);
std::vector<const Decl *> SelectedDecls = Symbols.Decls;

DocumentHighlightsFinder DocHighlightsFinder(
llvm::errs(), AST.getASTContext(), AST.getPreprocessor(), SelectedDecls);

index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
IndexOpts.IndexFunctionLocals = true;
indexTopLevelDecls(AST.getASTContext(), AST.getTopLevelDecls(),
DocHighlightsFinder, IndexOpts);

Expand Down Expand Up @@ -409,25 +532,14 @@ Hover getHover(ParsedAST &AST, Position Pos) {
const SourceManager &SourceMgr = AST.getASTContext().getSourceManager();
SourceLocation SourceLocationBeg =
getBeginningOfIdentifier(AST, Pos, SourceMgr.getMainFileID());
DeclarationAndMacrosFinder DeclMacrosFinder(llvm::errs(), SourceLocationBeg,
AST.getASTContext(),
AST.getPreprocessor());

index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
IndexOpts.IndexFunctionLocals = true;

indexTopLevelDecls(AST.getASTContext(), AST.getTopLevelDecls(),
DeclMacrosFinder, IndexOpts);
// Identified symbols at a specific position.
auto Symbols = getSymbolAtPosition(AST, SourceLocationBeg);

std::vector<MacroDecl> Macros = DeclMacrosFinder.takeMacroInfos();
if (!Macros.empty())
return getHoverContents(Macros[0].Name);
if (!Symbols.Macros.empty())
return getHoverContents(Symbols.Macros[0].Name);

std::vector<const Decl *> Decls = DeclMacrosFinder.takeDecls();
if (!Decls.empty())
return getHoverContents(Decls[0]);
if (!Symbols.Decls.empty())
return getHoverContents(Symbols.Decls[0]);

return Hover();
}
Expand Down
4 changes: 3 additions & 1 deletion clang-tools-extra/clangd/XRefs.h
View file
Expand Up @@ -15,13 +15,15 @@

#include "ClangdUnit.h"
#include "Protocol.h"
#include "index/Index.h"
#include <vector>

namespace clang {
namespace clangd {

/// Get definition of symbol at a specified \p Pos.
std::vector<Location> findDefinitions(ParsedAST &AST, Position Pos);
std::vector<Location> findDefinitions(ParsedAST &AST, Position Pos,
const SymbolIndex *Index = nullptr);

/// Returns highlights for all usages of a symbol at \p Pos.
std::vector<DocumentHighlight> findDocumentHighlights(ParsedAST &AST,
Expand Down
22 changes: 8 additions & 14 deletions clang-tools-extra/clangd/index/FileIndex.cpp
View file
Expand Up @@ -13,12 +13,9 @@

namespace clang {
namespace clangd {
namespace {

/// Retrieves namespace and class level symbols in \p Decls.
std::unique_ptr<SymbolSlab> indexAST(ASTContext &Ctx,
std::shared_ptr<Preprocessor> PP,
llvm::ArrayRef<const Decl *> Decls) {
SymbolSlab indexAST(ParsedAST *AST) {
assert(AST && "AST must not be nullptr!");
SymbolCollector::Options CollectorOpts;
// FIXME(ioeric): we might also want to collect include headers. We would need
// to make sure all includes are canonicalized (with CanonicalIncludes), which
Expand All @@ -29,21 +26,18 @@ std::unique_ptr<SymbolSlab> indexAST(ASTContext &Ctx,
CollectorOpts.CountReferences = false;

SymbolCollector Collector(std::move(CollectorOpts));
Collector.setPreprocessor(std::move(PP));
Collector.setPreprocessor(AST->getPreprocessorPtr());
index::IndexingOptions IndexOpts;
// We only need declarations, because we don't count references.
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
IndexOpts.IndexFunctionLocals = false;

index::indexTopLevelDecls(Ctx, Decls, Collector, IndexOpts);
auto Symbols = llvm::make_unique<SymbolSlab>();
*Symbols = Collector.takeSymbols();
return Symbols;
index::indexTopLevelDecls(AST->getASTContext(), AST->getTopLevelDecls(),
Collector, IndexOpts);
return Collector.takeSymbols();
}

} // namespace

void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Slab) {
std::lock_guard<std::mutex> Lock(Mutex);
if (!Slab)
Expand Down Expand Up @@ -79,8 +73,8 @@ void FileIndex::update(PathRef Path, ParsedAST *AST) {
if (!AST) {
FSymbols.update(Path, nullptr);
} else {
auto Slab = indexAST(AST->getASTContext(), AST->getPreprocessorPtr(),
AST->getTopLevelDecls());
auto Slab = llvm::make_unique<SymbolSlab>();
*Slab = indexAST(AST);
FSymbols.update(Path, std::move(Slab));
}
auto Symbols = FSymbols.allSymbols();
Expand Down
4 changes: 4 additions & 0 deletions clang-tools-extra/clangd/index/FileIndex.h
View file
Expand Up @@ -71,6 +71,10 @@ class FileIndex : public SymbolIndex {
MemIndex Index;
};

/// Retrieves namespace and class level symbols in \p AST.
/// Exposed to assist in unit tests.
SymbolSlab indexAST(ParsedAST *AST);

} // namespace clangd
} // namespace clang

Expand Down

0 comments on commit 88cfb66

Please sign in to comment.