Skip to content

Commit

Permalink
[clangd] Add semanticTokens modifiers for function/class/file/global …
Browse files Browse the repository at this point in the history 
…scope

These allow (function-) local variables to be distinguished, but also a
bunch more cases.
It's not quite independent with existing information (e.g. the
field/variable distinction is redundant if you have class-scope + static
attributes) but I don't think this is terribly important.

Depends on D77811

Differential Revision: https://reviews.llvm.org/D95701
  • Loading branch information
@sam-mccall
sam-mccall committed Feb 9, 2021
1 parent 399c3d5 commit 46cc7ce
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 33 deletions.
92 changes: 79 additions & 13 deletions clang-tools-extra/clangd/SemanticHighlighting.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,46 @@ class HighlightingsBuilder {
HighlightingToken Dummy; // returned from addToken(InvalidLoc)
};

llvm::Optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {
const DeclContext *DC = D->getDeclContext();
// Injected "Foo" within the class "Foo" has file scope, not class scope.
if (auto *R = dyn_cast_or_null<RecordDecl>(D))
if (R->isInjectedClassName())
DC = DC->getParent();
// Lambda captures are considered function scope, not class scope.
if (llvm::isa<FieldDecl>(D))
if (const auto *RD = llvm::dyn_cast<RecordDecl>(DC))
if (RD->isLambda())
return HighlightingModifier::FunctionScope;
// Walk up the DeclContext hierarchy until we find something interesting.
for (; !DC->isFileContext(); DC = DC->getParent()) {
if (DC->isFunctionOrMethod())
return HighlightingModifier::FunctionScope;
if (DC->isRecord())
return HighlightingModifier::ClassScope;
}
// Some template parameters (e.g. those for variable templates) don't have
// meaningful DeclContexts. That doesn't mean they're global!
if (DC->isTranslationUnit() && D->isTemplateParameter())
return llvm::None;
// ExternalLinkage threshold could be tweaked, e.g. module-visible as global.
if (D->getLinkageInternal() < ExternalLinkage)
return HighlightingModifier::FileScope;
return HighlightingModifier::GlobalScope;
}

llvm::Optional<HighlightingModifier> scopeModifier(const Type *T) {
if (!T)
return llvm::None;
if (T->isBuiltinType())
return HighlightingModifier::GlobalScope;
if (auto *TD = dyn_cast<TemplateTypeParmType>(T))
return scopeModifier(TD->getDecl());
if (auto *TD = T->getAsTagDecl())
return scopeModifier(TD);
return llvm::None;
}

/// Produces highlightings, which are not captured by findExplicitReferences,
/// e.g. highlights dependent names and 'auto' as the underlying type.
class CollectExtraHighlightings
Expand All @@ -365,55 +405,68 @@ class CollectExtraHighlightings
CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}

bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
if (auto K = kindForType(L.getTypePtr()))
H.addToken(L.getBeginLoc(), *K)
.addModifier(HighlightingModifier::Deduced);
if (auto K = kindForType(L.getTypePtr())) {
auto &Tok = H.addToken(L.getBeginLoc(), *K)
.addModifier(HighlightingModifier::Deduced);
if (auto Mod = scopeModifier(L.getTypePtr()))
Tok.addModifier(*Mod);
}
return true;
}

bool VisitDeclaratorDecl(DeclaratorDecl *D) {
auto *AT = D->getType()->getContainedAutoType();
if (!AT)
return true;
if (auto K = kindForType(AT->getDeducedType().getTypePtrOrNull()))
H.addToken(D->getTypeSpecStartLoc(), *K)
.addModifier(HighlightingModifier::Deduced);
if (auto K = kindForType(AT->getDeducedType().getTypePtrOrNull())) {
auto &Tok = H.addToken(D->getTypeSpecStartLoc(), *K)
.addModifier(HighlightingModifier::Deduced);
if (auto Mod = scopeModifier(AT->getDeducedType().getTypePtrOrNull()))
Tok.addModifier(*Mod);
}
return true;
}

bool VisitOverloadExpr(OverloadExpr *E) {
if (!E->decls().empty())
return true; // handled by findExplicitReferences.
H.addToken(E->getNameLoc(), HighlightingKind::DependentName);
auto &Tok = H.addToken(E->getNameLoc(), HighlightingKind::DependentName);
if (llvm::isa<UnresolvedMemberExpr>(E))
Tok.addModifier(HighlightingModifier::ClassScope);
// other case is UnresolvedLookupExpr, scope is unknown.
return true;
}

bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
H.addToken(E->getMemberNameInfo().getLoc(),
HighlightingKind::DependentName);
H.addToken(E->getMemberNameInfo().getLoc(), HighlightingKind::DependentName)
.addModifier(HighlightingModifier::ClassScope);
return true;
}

bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
H.addToken(E->getNameInfo().getLoc(), HighlightingKind::DependentName);
H.addToken(E->getNameInfo().getLoc(), HighlightingKind::DependentName)
.addModifier(HighlightingModifier::ClassScope);
return true;
}

bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
H.addToken(L.getNameLoc(), HighlightingKind::DependentType);
H.addToken(L.getNameLoc(), HighlightingKind::DependentType)
.addModifier(HighlightingModifier::ClassScope);
return true;
}

bool VisitDependentTemplateSpecializationTypeLoc(
DependentTemplateSpecializationTypeLoc L) {
H.addToken(L.getTemplateNameLoc(), HighlightingKind::DependentType);
H.addToken(L.getTemplateNameLoc(), HighlightingKind::DependentType)
.addModifier(HighlightingModifier::ClassScope);
return true;
}

bool TraverseTemplateArgumentLoc(TemplateArgumentLoc L) {
switch (L.getArgument().getKind()) {
case TemplateArgument::Template:
case TemplateArgument::TemplateExpansion:
// FIXME: this isn't *always* a dependent template name.
H.addToken(L.getTemplateNameLoc(), HighlightingKind::DependentType);
break;
default:
Expand All @@ -430,7 +483,8 @@ class CollectExtraHighlightings
bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
if (NNS->getKind() == NestedNameSpecifier::Identifier)
H.addToken(Q.getLocalBeginLoc(), HighlightingKind::DependentType);
H.addToken(Q.getLocalBeginLoc(), HighlightingKind::DependentType)
.addModifier(HighlightingModifier::ClassScope);
}
return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
}
Expand Down Expand Up @@ -484,6 +538,8 @@ std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
if (auto *Templated = TD->getTemplatedDecl())
Decl = Templated;
}
if (auto Mod = scopeModifier(Decl))
Tok.addModifier(*Mod);
if (isConst(Decl))
Tok.addModifier(HighlightingModifier::Readonly);
if (isStatic(Decl))
Expand All @@ -499,6 +555,7 @@ std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
// Add highlightings for macro references.
auto AddMacro = [&](const MacroOccurrence &M) {
auto &T = Builder.addToken(M.Rng, HighlightingKind::Macro);
T.addModifier(HighlightingModifier::GlobalScope);
if (M.IsDefinition)
T.addModifier(HighlightingModifier::Declaration);
};
Expand Down Expand Up @@ -724,7 +781,16 @@ llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
return "deduced"; // nonstandard
case HighlightingModifier::Abstract:
return "abstract";
case HighlightingModifier::FunctionScope:
return "functionScope"; // nonstandard
case HighlightingModifier::ClassScope:
return "classScope"; // nonstandard
case HighlightingModifier::FileScope:
return "fileScope"; // nonstandard
case HighlightingModifier::GlobalScope:
return "globalScope"; // nonstandard
}
llvm_unreachable("unhandled HighlightingModifier");
}

std::vector<TheiaSemanticHighlightingInformation>
Expand Down
7 changes: 6 additions & 1 deletion clang-tools-extra/clangd/SemanticHighlighting.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,12 @@ enum class HighlightingModifier {
Static,
Abstract,

LastModifier = Abstract
FunctionScope,
ClassScope,
FileScope,
GlobalScope,

LastModifier = GlobalScope
};
static_assert(static_cast<unsigned>(HighlightingModifier::LastModifier) < 32,
"Increase width of modifiers bitfield!");
Expand Down
4 changes: 4 additions & 0 deletions clang-tools-extra/clangd/test/initialize-params.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,10 @@
# CHECK-NEXT: "readonly",
# CHECK-NEXT: "static",
# CHECK-NEXT: "abstract"
# CHECK-NEXT: "functionScope",
# CHECK-NEXT: "classScope",
# CHECK-NEXT: "fileScope",
# CHECK-NEXT: "globalScope"
# CHECK-NEXT: ],
# CHECK-NEXT: "tokenTypes": [
# CHECK-NEXT: "variable",
Expand Down
12 changes: 6 additions & 6 deletions clang-tools-extra/clangd/test/semantic-tokens.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
# CHECK-NEXT: "jsonrpc": "2.0",
# CHECK-NEXT: "result": {
# CHECK-NEXT: "data": [
# First line, char 5, variable, declaration
# First line, char 5, variable, declaration+globalScope
# CHECK-NEXT: 0,
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
# CHECK-NEXT: 1
# CHECK-NEXT: 513
# CHECK-NEXT: ],
# CHECK-NEXT: "resultId": "1"
# CHECK-NEXT: }
Expand All @@ -44,12 +44,12 @@
# CHECK-NEXT: "edits": [
# CHECK-NEXT: {
# CHECK-NEXT: "data": [
# Next line, char 5, variable, declaration
# Next line, char 5, variable, declaration+globalScope
# CHECK-NEXT: 1,
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
# CHECK-NEXT: 1
# CHECK-NEXT: 513
# CHECK-NEXT: ],
# Inserted at position 1
# CHECK-NEXT: "deleteCount": 0,
Expand All @@ -72,12 +72,12 @@
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
# CHECK-NEXT: 1,
# CHECK-NEXT: 513,
# CHECK-NEXT: 1,
# CHECK-NEXT: 4,
# CHECK-NEXT: 1,
# CHECK-NEXT: 0,
# CHECK-NEXT: 1
# CHECK-NEXT: 513
# CHECK-NEXT: ],
# CHECK-NEXT: "resultId": "3"
# CHECK-NEXT: }
Expand Down
78 changes: 71 additions & 7 deletions clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ std::string annotate(llvm::StringRef Input,
void checkHighlightings(llvm::StringRef Code,
std::vector<std::pair</*FileName*/ llvm::StringRef,
/*FileContent*/ llvm::StringRef>>
AdditionalFiles = {}) {
AdditionalFiles = {},
uint32_t ModifierMask = -1) {
Annotations Test(Code);
TestTU TU;
TU.Code = std::string(Test.code());
Expand All @@ -126,8 +127,11 @@ void checkHighlightings(llvm::StringRef Code,
for (auto File : AdditionalFiles)
TU.AdditionalFiles.insert({File.first, std::string(File.second)});
auto AST = TU.build();
auto Actual = getSemanticHighlightings(AST);
for (auto &Token : Actual)
Token.Modifiers &= ModifierMask;

EXPECT_EQ(Code, annotate(Test.code(), getSemanticHighlightings(AST)));
EXPECT_EQ(Code, annotate(Test.code(), Actual));
}

// Any annotations in OldCode and NewCode are converted into their corresponding
Expand Down Expand Up @@ -163,6 +167,12 @@ void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
<< OldCode;
}

constexpr static uint32_t ScopeModifierMask =
1 << unsigned(HighlightingModifier::FunctionScope) |
1 << unsigned(HighlightingModifier::ClassScope) |
1 << unsigned(HighlightingModifier::FileScope) |
1 << unsigned(HighlightingModifier::GlobalScope);

TEST(SemanticHighlighting, GetsCorrectTokens) {
const char *TestCases[] = {
R"cpp(
Expand Down Expand Up @@ -720,9 +730,10 @@ sizeof...($TemplateParameter[[Elements]]);
<:[deprecated]:> int $Variable_decl_deprecated[[x]];
)cpp",
};
for (const auto &TestCase : TestCases) {
checkHighlightings(TestCase);
}
for (const auto &TestCase : TestCases)
// Mask off scope modifiers to keep the tests manageable.
// They're tested separately.
checkHighlightings(TestCase, {}, ~ScopeModifierMask);

checkHighlightings(R"cpp(
class $Class_decl[[A]] {
Expand All @@ -732,7 +743,8 @@ sizeof...($TemplateParameter[[Elements]]);
{{"imp.h", R"cpp(
int someMethod();
void otherMethod();
)cpp"}});
)cpp"}},
~ScopeModifierMask);

// A separate test for macros in headers.
checkHighlightings(R"cpp(
Expand All @@ -745,7 +757,59 @@ sizeof...($TemplateParameter[[Elements]]);
#define DXYZ_Y(Y) DXYZ(x##Y)
#define DEFINE(X) int X;
#define DEFINE_Y DEFINE(Y)
)cpp"}});
)cpp"}},
~ScopeModifierMask);
}

TEST(SemanticHighlighting, ScopeModifiers) {
const char *TestCases[] = {
R"cpp(
static int $Variable_fileScope[[x]];
namespace $Namespace_globalScope[[ns]] {
class $Class_globalScope[[x]];
}
namespace {
void $Function_fileScope[[foo]]();
}
)cpp",
R"cpp(
void $Function_globalScope[[foo]](int $Parameter_functionScope[[y]]) {
int $LocalVariable_functionScope[[z]];
}
)cpp",
R"cpp(
// Lambdas are considered functions, not classes.
auto $Variable_fileScope[[x]] = [m(42)] { // FIXME: annotate capture
return $LocalVariable_functionScope[[m]];
};
)cpp",
R"cpp(
// Classes in functions are classes.
void $Function_globalScope[[foo]]() {
class $Class_functionScope[[X]] {
int $Field_classScope[[x]];
};
};
)cpp",
R"cpp(
template <int $TemplateParameter_classScope[[T]]>
class $Class_globalScope[[X]] {
};
)cpp",
R"cpp(
// No useful scope for template parameters of variable templates.
template <typename $TemplateParameter[[A]]>
unsigned $Variable_globalScope[[X]] =
$TemplateParameter[[A]]::$DependentName_classScope[[x]];
)cpp",
R"cpp(
#define $Macro_globalScope[[X]] 1
int $Variable_globalScope[[Y]] = $Macro_globalScope[[X]];
)cpp",
};

for (const char *Test : TestCases)
checkHighlightings(Test, {}, ScopeModifierMask);
}

TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
Expand Down
13 changes: 7 additions & 6 deletions clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,18 @@ TWEAK_TEST(AnnotateHighlightings);
TEST_F(AnnotateHighlightingsTest, Test) {
EXPECT_AVAILABLE("^vo^id^ ^f(^) {^}^"); // available everywhere.
EXPECT_AVAILABLE("[[int a; int b;]]");
EXPECT_EQ("void /* Function [decl] */f() {}", apply("void ^f() {}"));
EXPECT_EQ("void /* Function [decl] [globalScope] */f() {}",
apply("void ^f() {}"));

EXPECT_EQ(
apply("[[int f1(); const int x = f1();]]"),
"int /* Function [decl] */f1(); "
"const int /* Variable [decl] [readonly] */x = /* Function */f1();");
EXPECT_EQ(apply("[[int f1(); const int x = f1();]]"),
"int /* Function [decl] [globalScope] */f1(); "
"const int /* Variable [decl] [readonly] [fileScope] */x = "
"/* Function [globalScope] */f1();");

// Only the targeted range is annotated.
EXPECT_EQ(apply("void f1(); void f2() {^}"),
"void f1(); "
"void /* Function [decl] */f2() {}");
"void /* Function [decl] [globalScope] */f2() {}");
}

} // namespace
Expand Down

0 comments on commit 46cc7ce

Please sign in to comment.