142 changes: 142 additions & 0 deletions bolt/unittests/Core/MemoryMaps.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
//===- bolt/unittest/Core/MemoryMaps.cpp ----------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "bolt/Core/BinaryContext.h"
#include "bolt/Profile/DataAggregator.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Testing/Support/Error.h"
#include "gtest/gtest.h"

using namespace llvm;
using namespace llvm::object;
using namespace llvm::ELF;
using namespace bolt;

namespace opts {
extern cl::opt<std::string> ReadPerfEvents;
} // namespace opts

namespace {

/// Perform checks on memory map events normally captured in perf. Tests use
/// the 'opts::ReadPerfEvents' flag to emulate these events, passing a custom
/// 'perf script' output to DataAggregator.
struct MemoryMapsTester : public testing::TestWithParam<Triple::ArchType> {
void SetUp() override {
initalizeLLVM();
prepareElf();
initializeBOLT();
}

protected:
void initalizeLLVM() {
llvm::InitializeAllTargetInfos();
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmParsers();
llvm::InitializeAllDisassemblers();
llvm::InitializeAllTargets();
llvm::InitializeAllAsmPrinters();
}

void prepareElf() {
memcpy(ElfBuf, "\177ELF", 4);
ELF64LE::Ehdr *EHdr = reinterpret_cast<typename ELF64LE::Ehdr *>(ElfBuf);
EHdr->e_ident[llvm::ELF::EI_CLASS] = llvm::ELF::ELFCLASS64;
EHdr->e_ident[llvm::ELF::EI_DATA] = llvm::ELF::ELFDATA2LSB;
EHdr->e_machine = GetParam() == Triple::aarch64 ? EM_AARCH64 : EM_X86_64;
MemoryBufferRef Source(StringRef(ElfBuf, sizeof(ElfBuf)), "ELF");
ObjFile = cantFail(ObjectFile::createObjectFile(Source));
}

void initializeBOLT() {
Relocation::Arch = ObjFile->makeTriple().getArch();
BC = cantFail(BinaryContext::createBinaryContext(
ObjFile->makeTriple(), ObjFile->getFileName(), nullptr, true,
DWARFContext::create(*ObjFile.get()), {llvm::outs(), llvm::errs()}));
ASSERT_FALSE(!BC);
}

char ElfBuf[sizeof(typename ELF64LE::Ehdr)] = {};
std::unique_ptr<ObjectFile> ObjFile;
std::unique_ptr<BinaryContext> BC;
};
} // namespace

#ifdef X86_AVAILABLE

INSTANTIATE_TEST_SUITE_P(X86, MemoryMapsTester,
::testing::Values(Triple::x86_64));

#endif

#ifdef AARCH64_AVAILABLE

INSTANTIATE_TEST_SUITE_P(AArch64, MemoryMapsTester,
::testing::Values(Triple::aarch64));

#endif

/// Check that the correct mmap size is computed when we have multiple text
/// segment mappings.
TEST_P(MemoryMapsTester, ParseMultipleSegments) {
const int Pid = 1234;
StringRef Filename = "BINARY";
opts::ReadPerfEvents = formatv(
"name 0 [000] 0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
"[0xabc0000000(0x1000000) @ 0x11c0000 103:01 1573523 0]: r-xp {1}\n"
"name 0 [000] 0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
"[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
Pid, Filename);

BC->SegmentMapInfo[0x11da000] =
SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
BC->SegmentMapInfo[0x31d0000] =
SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0000, 0x3000000, 0x200000, true};

DataAggregator DA("");
BC->setFilename(Filename);
Error Err = DA.preprocessProfile(*BC);

// Ignore errors from perf2bolt when parsing memory events later on.
ASSERT_THAT_ERROR(std::move(Err), Succeeded());

auto &BinaryMMapInfo = DA.getBinaryMMapInfo();
auto El = BinaryMMapInfo.find(Pid);
// Check that memory mapping is present and has the expected size.
ASSERT_NE(El, BinaryMMapInfo.end());
ASSERT_EQ(El->second.Size, static_cast<uint64_t>(0xb1d0000));
}

/// Check that DataAggregator aborts when pre-processing an input binary
/// with multiple text segments that have different base addresses.
TEST_P(MemoryMapsTester, MultipleSegmentsMismatchedBaseAddress) {
const int Pid = 1234;
StringRef Filename = "BINARY";
opts::ReadPerfEvents = formatv(
"name 0 [000] 0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
"[0xabc0000000(0x1000000) @ 0x11c0000 103:01 1573523 0]: r-xp {1}\n"
"name 0 [000] 0.000000: PERF_RECORD_MMAP2 {0}/{0}: "
"[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
Pid, Filename);

BC->SegmentMapInfo[0x11da000] =
SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
// Using '0x31d0fff' FileOffset which triggers a different base address
// for this second text segment.
BC->SegmentMapInfo[0x31d0000] =
SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0fff, 0x3000000, 0x200000, true};

DataAggregator DA("");
BC->setFilename(Filename);
ASSERT_DEBUG_DEATH(
{ Error Err = DA.preprocessProfile(*BC); },
"Base address on multiple segment mappings should match");
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//

#include "ReturnConstRefFromParameterCheck.h"
#include "clang/AST/Attrs.inc"
#include "clang/AST/Expr.h"
#include "clang/ASTMatchers/ASTMatchFinder.h"
#include "clang/ASTMatchers/ASTMatchers.h"
Expand All @@ -15,14 +16,23 @@ using namespace clang::ast_matchers;

namespace clang::tidy::bugprone {

namespace {

AST_MATCHER(ParmVarDecl, hasLifetimeBoundAttr) {
return Node.hasAttr<LifetimeBoundAttr>();
}

} // namespace

void ReturnConstRefFromParameterCheck::registerMatchers(MatchFinder *Finder) {
const auto DRef = ignoringParens(
declRefExpr(
to(parmVarDecl(hasType(hasCanonicalType(
qualType(lValueReferenceType(pointee(
qualType(isConstQualified()))))
.bind("type"))),
hasDeclContext(functionDecl().bind("owner")))
hasDeclContext(functionDecl().bind("owner")),
unless(hasLifetimeBoundAttr()))
.bind("param")))
.bind("dref"));
const auto Func =
Expand Down
7 changes: 5 additions & 2 deletions clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,11 @@ void UseInternalLinkageCheck::registerMatchers(MatchFinder *Finder) {
isExternStorageClass(), isExternC(),
// 3. template
isExplicitTemplateSpecialization(),
// 4. friend
hasAncestor(friendDecl()))));
hasAncestor(decl(anyOf(
// 4. friend
friendDecl(),
// 5. module export decl
exportDecl()))))));
Finder->addMatcher(
functionDecl(Common, hasBody(),
unless(anyOf(cxxMethodDecl(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,10 @@ namespace clang::tidy::readability {
/// a call to `empty()`.
///
/// The emptiness of a container should be checked using the `empty()` method
/// instead of the `size()` method. It is not guaranteed that `size()` is a
/// constant-time function, and it is generally more efficient and also shows
/// clearer intent to use `empty()`. Furthermore some containers may implement
/// the `empty()` method but not implement the `size()` method. Using `empty()`
/// whenever possible makes it easier to switch to another container in the
/// future.
/// instead of the `size()` method. It shows clearer intent to use `empty()`.
/// Furthermore some containers may implement the `empty()` method but not
/// implement the `size()` method. Using `empty()` whenever possible makes it
/// easier to switch to another container in the future.
class ContainerSizeEmptyCheck : public ClangTidyCheck {
public:
ContainerSizeEmptyCheck(StringRef Name, ClangTidyContext *Context);
Expand Down
8 changes: 8 additions & 0 deletions clang-tools-extra/clangd/ClangdLSPServer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1415,6 +1415,12 @@ void ClangdLSPServer::onInlayHint(const InlayHintsParams &Params,
std::move(Reply));
}

void ClangdLSPServer::onCallHierarchyOutgoingCalls(
const CallHierarchyOutgoingCallsParams &Params,
Callback<std::vector<CallHierarchyOutgoingCall>> Reply) {
Server->outgoingCalls(Params.item, std::move(Reply));
}

void ClangdLSPServer::applyConfiguration(
const ConfigurationSettings &Settings) {
// Per-file update to the compilation database.
Expand Down Expand Up @@ -1693,6 +1699,8 @@ void ClangdLSPServer::bindMethods(LSPBinder &Bind,
Bind.method("typeHierarchy/subtypes", this, &ClangdLSPServer::onSubTypes);
Bind.method("textDocument/prepareCallHierarchy", this, &ClangdLSPServer::onPrepareCallHierarchy);
Bind.method("callHierarchy/incomingCalls", this, &ClangdLSPServer::onCallHierarchyIncomingCalls);
if (Opts.EnableOutgoingCalls)
Bind.method("callHierarchy/outgoingCalls", this, &ClangdLSPServer::onCallHierarchyOutgoingCalls);
Bind.method("textDocument/selectionRange", this, &ClangdLSPServer::onSelectionRange);
Bind.method("textDocument/documentLink", this, &ClangdLSPServer::onDocumentLink);
Bind.method("textDocument/semanticTokens/full", this, &ClangdLSPServer::onSemanticTokens);
Expand Down
3 changes: 3 additions & 0 deletions clang-tools-extra/clangd/ClangdLSPServer.h
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
void onCallHierarchyIncomingCalls(
const CallHierarchyIncomingCallsParams &,
Callback<std::vector<CallHierarchyIncomingCall>>);
void onCallHierarchyOutgoingCalls(
const CallHierarchyOutgoingCallsParams &,
Callback<std::vector<CallHierarchyOutgoingCall>>);
void onClangdInlayHints(const InlayHintsParams &,
Callback<llvm::json::Value>);
void onInlayHint(const InlayHintsParams &, Callback<std::vector<InlayHint>>);
Expand Down
14 changes: 13 additions & 1 deletion clang-tools-extra/clangd/ClangdServer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,9 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
const ThreadsafeFS &TFS, const Options &Opts,
Callbacks *Callbacks)
: FeatureModules(Opts.FeatureModules), CDB(CDB), TFS(TFS),
DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr),
DynamicIdx(Opts.BuildDynamicSymbolIndex
? new FileIndex(Opts.EnableOutgoingCalls)
: nullptr),
ModulesManager(Opts.ModulesManager),
ClangTidyProvider(Opts.ClangTidyProvider),
UseDirtyHeaders(Opts.UseDirtyHeaders),
Expand Down Expand Up @@ -256,6 +258,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
Callbacks->onBackgroundIndexProgress(S);
};
BGOpts.ContextProvider = Opts.ContextProvider;
BGOpts.SupportContainedRefs = Opts.EnableOutgoingCalls;
BackgroundIdx = std::make_unique<BackgroundIndex>(
TFS, CDB,
BackgroundIndexStorage::createDiskBackedStorageFactory(
Expand Down Expand Up @@ -912,6 +915,15 @@ void ClangdServer::inlayHints(PathRef File, std::optional<Range> RestrictRange,
WorkScheduler->runWithAST("InlayHints", File, std::move(Action), Transient);
}

void ClangdServer::outgoingCalls(
const CallHierarchyItem &Item,
Callback<std::vector<CallHierarchyOutgoingCall>> CB) {
WorkScheduler->run("Outgoing Calls", "",
[CB = std::move(CB), Item, this]() mutable {
CB(clangd::outgoingCalls(Item, Index));
});
}

void ClangdServer::onFileEvent(const DidChangeWatchedFilesParams &Params) {
// FIXME: Do nothing for now. This will be used for indexing and potentially
// invalidating other caches.
Expand Down
9 changes: 9 additions & 0 deletions clang-tools-extra/clangd/ClangdServer.h
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,11 @@ class ClangdServer {
/// Cached preambles are potentially large. If false, store them on disk.
bool StorePreamblesInMemory = true;

/// Call hierarchy's outgoing calls feature requires additional index
/// serving structures which increase memory usage. If false, these are
/// not created and the feature is not enabled.
bool EnableOutgoingCalls = true;

/// This throttler controls which preambles may be built at a given time.
clangd::PreambleThrottler *PreambleThrottler = nullptr;

Expand Down Expand Up @@ -292,6 +297,10 @@ class ClangdServer {
void incomingCalls(const CallHierarchyItem &Item,
Callback<std::vector<CallHierarchyIncomingCall>>);

/// Resolve outgoing calls for a given call hierarchy item.
void outgoingCalls(const CallHierarchyItem &Item,
Callback<std::vector<CallHierarchyOutgoingCall>>);

/// Resolve inlay hints for a given document.
void inlayHints(PathRef File, std::optional<Range> RestrictRange,
Callback<std::vector<InlayHint>>);
Expand Down
61 changes: 61 additions & 0 deletions clang-tools-extra/clangd/XRefs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1702,6 +1702,7 @@ declToHierarchyItem(const NamedDecl &ND, llvm::StringRef TUPath) {

HierarchyItem HI;
HI.name = printName(Ctx, ND);
// FIXME: Populate HI.detail the way we do in symbolToHierarchyItem?
HI.kind = SK;
HI.range = Range{sourceLocToPosition(SM, DeclRange->getBegin()),
sourceLocToPosition(SM, DeclRange->getEnd())};
Expand Down Expand Up @@ -1753,6 +1754,7 @@ static std::optional<HierarchyItem> symbolToHierarchyItem(const Symbol &S,
}
HierarchyItem HI;
HI.name = std::string(S.Name);
HI.detail = (S.Scope + S.Name).str();
HI.kind = indexSymbolKindToSymbolKind(S.SymInfo.Kind);
HI.selectionRange = Loc->range;
// FIXME: Populate 'range' correctly
Expand Down Expand Up @@ -2319,6 +2321,65 @@ incomingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index) {
return Results;
}

std::vector<CallHierarchyOutgoingCall>
outgoingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index) {
std::vector<CallHierarchyOutgoingCall> Results;
if (!Index || Item.data.empty())
return Results;
auto ID = SymbolID::fromStr(Item.data);
if (!ID) {
elog("outgoingCalls failed to find symbol: {0}", ID.takeError());
return Results;
}
// In this function, we find outgoing calls based on the index only.
ContainedRefsRequest Request;
Request.ID = *ID;
// Initially store the ranges in a map keyed by SymbolID of the callee.
// This allows us to group different calls to the same function
// into the same CallHierarchyOutgoingCall.
llvm::DenseMap<SymbolID, std::vector<Range>> CallsOut;
// We can populate the ranges based on a refs request only. As we do so, we
// also accumulate the callee IDs into a lookup request.
LookupRequest CallsOutLookup;
Index->containedRefs(Request, [&](const auto &R) {
auto Loc = indexToLSPLocation(R.Location, Item.uri.file());
if (!Loc) {
elog("outgoingCalls failed to convert location: {0}", Loc.takeError());
return;
}
auto It = CallsOut.try_emplace(R.Symbol, std::vector<Range>{}).first;
It->second.push_back(Loc->range);

CallsOutLookup.IDs.insert(R.Symbol);
});
// Perform the lookup request and combine its results with CallsOut to
// get complete CallHierarchyOutgoingCall objects.
Index->lookup(CallsOutLookup, [&](const Symbol &Callee) {
// The containedRefs request should only return symbols which are
// function-like, i.e. symbols for which references to them can be "calls".
using SK = index::SymbolKind;
auto Kind = Callee.SymInfo.Kind;
assert(Kind == SK::Function || Kind == SK::InstanceMethod ||
Kind == SK::ClassMethod || Kind == SK::StaticMethod ||
Kind == SK::Constructor || Kind == SK::Destructor ||
Kind == SK::ConversionFunction);
(void)Kind;
(void)SK::Function;

auto It = CallsOut.find(Callee.ID);
assert(It != CallsOut.end());
if (auto CHI = symbolToCallHierarchyItem(Callee, Item.uri.file()))
Results.push_back(
CallHierarchyOutgoingCall{std::move(*CHI), std::move(It->second)});
});
// Sort results by name of the callee.
llvm::sort(Results, [](const CallHierarchyOutgoingCall &A,
const CallHierarchyOutgoingCall &B) {
return A.to.name < B.to.name;
});
return Results;
}

llvm::DenseSet<const Decl *> getNonLocalDeclRefs(ParsedAST &AST,
const FunctionDecl *FD) {
if (!FD->hasBody())
Expand Down
3 changes: 3 additions & 0 deletions clang-tools-extra/clangd/XRefs.h
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ prepareCallHierarchy(ParsedAST &AST, Position Pos, PathRef TUPath);
std::vector<CallHierarchyIncomingCall>
incomingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index);

std::vector<CallHierarchyOutgoingCall>
outgoingCalls(const CallHierarchyItem &Item, const SymbolIndex *Index);

/// Returns all decls that are referenced in the \p FD except local symbols.
llvm::DenseSet<const Decl *> getNonLocalDeclRefs(ParsedAST &AST,
const FunctionDecl *FD);
Expand Down
2 changes: 1 addition & 1 deletion clang-tools-extra/clangd/index/Background.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ BackgroundIndex::BackgroundIndex(
: SwapIndex(std::make_unique<MemIndex>()), TFS(TFS), CDB(CDB),
IndexingPriority(Opts.IndexingPriority),
ContextProvider(std::move(Opts.ContextProvider)),
IndexedSymbols(IndexContents::All),
IndexedSymbols(IndexContents::All, Opts.SupportContainedRefs),
Rebuilder(this, &IndexedSymbols, Opts.ThreadPoolSize),
IndexStorageFactory(std::move(IndexStorageFactory)),
Queue(std::move(Opts.OnProgress)),
Expand Down
3 changes: 3 additions & 0 deletions clang-tools-extra/clangd/index/Background.h
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,9 @@ class BackgroundIndex : public SwapIndex {
// file. Called with the empty string for other tasks.
// (When called, the context from BackgroundIndex construction is active).
std::function<Context(PathRef)> ContextProvider = nullptr;
// Whether the index needs to support the containedRefs() operation.
// May use extra memory.
bool SupportContainedRefs = true;
};

/// Creates a new background index and starts its threads.
Expand Down
2 changes: 1 addition & 1 deletion clang-tools-extra/clangd/index/BackgroundRebuild.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//===-- BackgroundRebuild.cpp - when to rebuild thei background index -----===//
//===-- BackgroundRebuild.cpp - when to rebuild the background index ------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
Expand Down
13 changes: 7 additions & 6 deletions clang-tools-extra/clangd/index/FileIndex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,8 @@ SlabTuple indexHeaderSymbols(llvm::StringRef Version, ASTContext &AST,
/*CollectMainFileRefs=*/false);
}

FileSymbols::FileSymbols(IndexContents IdxContents)
: IdxContents(IdxContents) {}
FileSymbols::FileSymbols(IndexContents IdxContents, bool SupportContainedRefs)
: IdxContents(IdxContents), SupportContainedRefs(SupportContainedRefs) {}

void FileSymbols::update(llvm::StringRef Key,
std::unique_ptr<SymbolSlab> Symbols,
Expand Down Expand Up @@ -395,7 +395,7 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
std::move(AllRelations), std::move(Files), IdxContents,
std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
std::move(RefsStorage), std::move(SymsStorage)),
StorageSize);
StorageSize, SupportContainedRefs);
}
llvm_unreachable("Unknown clangd::IndexType");
}
Expand All @@ -419,11 +419,12 @@ void FileSymbols::profile(MemoryTree &MT) const {
}
}

FileIndex::FileIndex()
FileIndex::FileIndex(bool SupportContainedRefs)
: MergedIndex(&MainFileIndex, &PreambleIndex),
PreambleSymbols(IndexContents::Symbols | IndexContents::Relations),
PreambleSymbols(IndexContents::Symbols | IndexContents::Relations,
SupportContainedRefs),
PreambleIndex(std::make_unique<MemIndex>()),
MainFileSymbols(IndexContents::All),
MainFileSymbols(IndexContents::All, SupportContainedRefs),
MainFileIndex(std::make_unique<MemIndex>()) {}

void FileIndex::updatePreamble(IndexFileIn IF) {
Expand Down
5 changes: 3 additions & 2 deletions clang-tools-extra/clangd/index/FileIndex.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ enum class DuplicateHandling {
/// locking when we swap or obtain references to snapshots.
class FileSymbols {
public:
FileSymbols(IndexContents IdxContents);
FileSymbols(IndexContents IdxContents, bool SupportContainedRefs);
/// Updates all slabs associated with the \p Key.
/// If either is nullptr, corresponding data for \p Key will be removed.
/// If CountReferences is true, \p Refs will be used for counting references
Expand All @@ -91,6 +91,7 @@ class FileSymbols {

private:
IndexContents IdxContents;
bool SupportContainedRefs;

struct RefSlabAndCountReferences {
std::shared_ptr<RefSlab> Slab;
Expand All @@ -108,7 +109,7 @@ class FileSymbols {
/// FIXME: Expose an interface to remove files that are closed.
class FileIndex : public MergedIndex {
public:
FileIndex();
FileIndex(bool SupportContainedRefs);

/// Update preamble symbols of file \p Path with all declarations in \p AST
/// and macros in \p PP.
Expand Down
5 changes: 5 additions & 0 deletions clang-tools-extra/clangd/index/Index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@ bool SwapIndex::refs(const RefsRequest &R,
llvm::function_ref<void(const Ref &)> CB) const {
return snapshot()->refs(R, CB);
}
bool SwapIndex::containedRefs(
const ContainedRefsRequest &R,
llvm::function_ref<void(const ContainedRefsResult &)> CB) const {
return snapshot()->containedRefs(R, CB);
}
void SwapIndex::relations(
const RelationsRequest &R,
llvm::function_ref<void(const SymbolID &, const Symbol &)> CB) const {
Expand Down
35 changes: 35 additions & 0 deletions clang-tools-extra/clangd/index/Index.h
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,34 @@ struct RefsRequest {
bool WantContainer = false;
};

struct ContainedRefsRequest {
/// Note that RefKind::Call just restricts the matched SymbolKind to
/// functions, not the form of the reference (e.g. address-of-function,
/// which can indicate an indirect call, should still be caught).
static const RefKind SupportedRefKinds = RefKind::Call;

SymbolID ID;
/// If set, limit the number of refers returned from the index. The index may
/// choose to return less than this, e.g. it tries to avoid returning stale
/// results.
std::optional<uint32_t> Limit;
};

struct RelationsRequest {
llvm::DenseSet<SymbolID> Subjects;
RelationKind Predicate;
/// If set, limit the number of relations returned from the index.
std::optional<uint32_t> Limit;
};

struct ContainedRefsResult {
/// The source location where the symbol is named.
SymbolLocation Location;
RefKind Kind = RefKind::Unknown;
/// The ID of the symbol which is referred to
SymbolID Symbol;
};

/// Describes what data is covered by an index.
///
/// Indexes may contain symbols but not references from a file, etc.
Expand Down Expand Up @@ -141,6 +162,17 @@ class SymbolIndex {
virtual bool refs(const RefsRequest &Req,
llvm::function_ref<void(const Ref &)> Callback) const = 0;

/// Find all symbols that are referenced by a symbol and apply
/// \p Callback on each result.
///
/// Results should be returned in arbitrary order.
/// The returned result must be deep-copied if it's used outside Callback.
///
/// Returns true if there will be more results (limited by Req.Limit);
virtual bool containedRefs(
const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)> Callback) const = 0;

/// Finds all relations (S, P, O) stored in the index such that S is among
/// Req.Subjects and P is Req.Predicate, and invokes \p Callback for (S, O) in
/// each.
Expand Down Expand Up @@ -175,6 +207,9 @@ class SwapIndex : public SymbolIndex {
llvm::function_ref<void(const Symbol &)>) const override;
bool refs(const RefsRequest &,
llvm::function_ref<void(const Ref &)>) const override;
bool containedRefs(
const ContainedRefsRequest &,
llvm::function_ref<void(const ContainedRefsResult &)>) const override;
void relations(const RelationsRequest &,
llvm::function_ref<void(const SymbolID &, const Symbol &)>)
const override;
Expand Down
20 changes: 20 additions & 0 deletions clang-tools-extra/clangd/index/MemIndex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "MemIndex.h"
#include "FuzzyMatch.h"
#include "Quality.h"
#include "index/Index.h"
#include "support/Trace.h"

namespace clang {
Expand Down Expand Up @@ -85,6 +86,25 @@ bool MemIndex::refs(const RefsRequest &Req,
return false; // We reported all refs.
}

bool MemIndex::containedRefs(
const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
trace::Span Tracer("MemIndex refersTo");
uint32_t Remaining = Req.Limit.value_or(std::numeric_limits<uint32_t>::max());
for (const auto &Pair : Refs) {
for (const auto &R : Pair.second) {
if (!static_cast<int>(ContainedRefsRequest::SupportedRefKinds & R.Kind) ||
Req.ID != R.Container)
continue;
if (Remaining == 0)
return true; // More refs were available.
--Remaining;
Callback({R.Location, R.Kind, Pair.first});
}
}
return false; // We reported all refs.
}

void MemIndex::relations(
const RelationsRequest &Req,
llvm::function_ref<void(const SymbolID &, const Symbol &)> Callback) const {
Expand Down
4 changes: 4 additions & 0 deletions clang-tools-extra/clangd/index/MemIndex.h
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ class MemIndex : public SymbolIndex {
bool refs(const RefsRequest &Req,
llvm::function_ref<void(const Ref &)> Callback) const override;

bool containedRefs(const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)>
Callback) const override;

void relations(const RelationsRequest &Req,
llvm::function_ref<void(const SymbolID &, const Symbol &)>
Callback) const override;
Expand Down
34 changes: 34 additions & 0 deletions clang-tools-extra/clangd/index/Merge.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,40 @@ bool MergedIndex::refs(const RefsRequest &Req,
return More || StaticHadMore;
}

bool MergedIndex::containedRefs(
const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
trace::Span Tracer("MergedIndex refersTo");
bool More = false;
uint32_t Remaining = Req.Limit.value_or(std::numeric_limits<uint32_t>::max());
// We don't want duplicated refs from the static/dynamic indexes,
// and we can't reliably deduplicate them because offsets may differ slightly.
// We consider the dynamic index authoritative and report all its refs,
// and only report static index refs from other files.
More |= Dynamic->containedRefs(Req, [&](const auto &O) {
Callback(O);
assert(Remaining != 0);
--Remaining;
});
if (Remaining == 0 && More)
return More;
auto DynamicContainsFile = Dynamic->indexedFiles();
// We return less than Req.Limit if static index returns more refs for dirty
// files.
bool StaticHadMore = Static->containedRefs(Req, [&](const auto &O) {
if ((DynamicContainsFile(O.Location.FileURI) & IndexContents::References) !=
IndexContents::None)
return; // ignore refs that have been seen from dynamic index.
if (Remaining == 0) {
More = true;
return;
}
--Remaining;
Callback(O);
});
return More || StaticHadMore;
}

llvm::unique_function<IndexContents(llvm::StringRef) const>
MergedIndex::indexedFiles() const {
return [DynamicContainsFile{Dynamic->indexedFiles()},
Expand Down
3 changes: 3 additions & 0 deletions clang-tools-extra/clangd/index/Merge.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ class MergedIndex : public SymbolIndex {
llvm::function_ref<void(const Symbol &)>) const override;
bool refs(const RefsRequest &,
llvm::function_ref<void(const Ref &)>) const override;
bool containedRefs(
const ContainedRefsRequest &,
llvm::function_ref<void(const ContainedRefsResult &)>) const override;
void relations(const RelationsRequest &,
llvm::function_ref<void(const SymbolID &, const Symbol &)>)
const override;
Expand Down
13 changes: 13 additions & 0 deletions clang-tools-extra/clangd/index/ProjectAware.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ class ProjectAwareIndex : public SymbolIndex {
/// Query all indexes while prioritizing the associated one (if any).
bool refs(const RefsRequest &Req,
llvm::function_ref<void(const Ref &)> Callback) const override;
/// Query all indexes while prioritizing the associated one (if any).
bool containedRefs(const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)>
Callback) const override;

/// Queries only the associates index when Req.RestrictForCodeCompletion is
/// set, otherwise queries all.
Expand Down Expand Up @@ -94,6 +98,15 @@ bool ProjectAwareIndex::refs(
return false;
}

bool ProjectAwareIndex::containedRefs(
const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
trace::Span Tracer("ProjectAwareIndex::refersTo");
if (auto *Idx = getIndex())
return Idx->containedRefs(Req, Callback);
return false;
}

bool ProjectAwareIndex::fuzzyFind(
const FuzzyFindRequest &Req,
llvm::function_ref<void(const Symbol &)> Callback) const {
Expand Down
3 changes: 3 additions & 0 deletions clang-tools-extra/clangd/index/Ref.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ enum class RefKind : uint8_t {
// ^ this references Foo, but does not explicitly spell out its name
// };
Spelled = 1 << 3,
// A reference which is a call. Used as a filter for which references
// to store in data structures used for computing outgoing calls.
Call = 1 << 4,
All = Declaration | Definition | Reference | Spelled,
};

Expand Down
14 changes: 8 additions & 6 deletions clang-tools-extra/clangd/index/Serialization.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ readCompileCommand(Reader CmdReader, llvm::ArrayRef<llvm::StringRef> Strings) {
// The current versioning scheme is simple - non-current versions are rejected.
// If you make a breaking change, bump this version number to invalidate stored
// data. Later we may want to support some backward compatibility.
constexpr static uint32_t Version = 19;
constexpr static uint32_t Version = 20;

llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data,
SymbolOrigin Origin) {
Expand Down Expand Up @@ -704,7 +704,8 @@ llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data,
}

std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
SymbolOrigin Origin, bool UseDex) {
SymbolOrigin Origin, bool UseDex,
bool SupportContainedRefs) {
trace::Span OverallTracer("LoadIndex");
auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
if (!Buffer) {
Expand Down Expand Up @@ -735,10 +736,11 @@ std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
size_t NumRelations = Relations.size();

trace::Span Tracer("BuildIndex");
auto Index = UseDex ? dex::Dex::build(std::move(Symbols), std::move(Refs),
std::move(Relations))
: MemIndex::build(std::move(Symbols), std::move(Refs),
std::move(Relations));
auto Index = UseDex
? dex::Dex::build(std::move(Symbols), std::move(Refs),
std::move(Relations), SupportContainedRefs)
: MemIndex::build(std::move(Symbols), std::move(Refs),
std::move(Relations));
vlog("Loaded {0} from {1} with estimated memory usage {2} bytes\n"
" - number of symbols: {3}\n"
" - number of refs: {4}\n"
Expand Down
3 changes: 2 additions & 1 deletion clang-tools-extra/clangd/index/Serialization.h
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ std::string toYAML(const Ref &);
// Build an in-memory static index from an index file.
// The size should be relatively small, so data can be managed in memory.
std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef Filename,
SymbolOrigin Origin, bool UseDex = true);
SymbolOrigin Origin, bool UseDex,
bool SupportContainedRefs);

} // namespace clangd
} // namespace clang
Expand Down
20 changes: 17 additions & 3 deletions clang-tools-extra/clangd/index/SymbolCollector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "clang-include-cleaner/Record.h"
#include "clang-include-cleaner/Types.h"
#include "index/CanonicalIncludes.h"
#include "index/Ref.h"
#include "index/Relation.h"
#include "index/Symbol.h"
#include "index/SymbolID.h"
Expand Down Expand Up @@ -660,7 +661,7 @@ bool SymbolCollector::handleDeclOccurrence(
auto FileLoc = SM.getFileLoc(Loc);
auto FID = SM.getFileID(FileLoc);
if (Opts.RefsInHeaders || FID == SM.getMainFileID()) {
addRef(ID, SymbolRef{FileLoc, FID, Roles,
addRef(ID, SymbolRef{FileLoc, FID, Roles, index::getSymbolInfo(ND).Kind,
getRefContainer(ASTNode.Parent, Opts),
isSpelled(FileLoc, *ND)});
}
Expand Down Expand Up @@ -774,8 +775,10 @@ bool SymbolCollector::handleMacroOccurrence(const IdentifierInfo *Name,
// FIXME: Populate container information for macro references.
// FIXME: All MacroRefs are marked as Spelled now, but this should be
// checked.
addRef(ID, SymbolRef{Loc, SM.getFileID(Loc), Roles, /*Container=*/nullptr,
/*Spelled=*/true});
addRef(ID,
SymbolRef{Loc, SM.getFileID(Loc), Roles, index::SymbolKind::Macro,
/*Container=*/nullptr,
/*Spelled=*/true});
}

// Collect symbols.
Expand Down Expand Up @@ -1166,6 +1169,14 @@ bool SymbolCollector::shouldIndexFile(FileID FID) {
return I.first->second;
}

static bool refIsCall(index::SymbolKind Kind) {
using SK = index::SymbolKind;
return Kind == SK::Function || Kind == SK::InstanceMethod ||
Kind == SK::ClassMethod || Kind == SK::StaticMethod ||
Kind == SK::Constructor || Kind == SK::Destructor ||
Kind == SK::ConversionFunction;
}

void SymbolCollector::addRef(SymbolID ID, const SymbolRef &SR) {
const auto &SM = ASTCtx->getSourceManager();
// FIXME: use the result to filter out references.
Expand All @@ -1177,6 +1188,9 @@ void SymbolCollector::addRef(SymbolID ID, const SymbolRef &SR) {
R.Location.End = Range.second;
R.Location.FileURI = HeaderFileURIs->toURI(*FE).c_str();
R.Kind = toRefKind(SR.Roles, SR.Spelled);
if (refIsCall(SR.Kind)) {
R.Kind |= RefKind::Call;
}
R.Container = getSymbolIDCached(SR.Container);
Refs.insert(ID, R);
}
Expand Down
1 change: 1 addition & 0 deletions clang-tools-extra/clangd/index/SymbolCollector.h
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ class SymbolCollector : public index::IndexDataConsumer {
SourceLocation Loc;
FileID FID;
index::SymbolRoleSet Roles;
index::SymbolKind Kind;
const Decl *Container;
bool Spelled;
};
Expand Down
52 changes: 49 additions & 3 deletions clang-tools-extra/clangd/index/dex/Dex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,14 @@ namespace clangd {
namespace dex {

std::unique_ptr<SymbolIndex> Dex::build(SymbolSlab Symbols, RefSlab Refs,
RelationSlab Rels) {
RelationSlab Rels,
bool SupportContainedRefs) {
auto Size = Symbols.bytes() + Refs.bytes();
// There is no need to include "Rels" in Data because the relations are self-
// contained, without references into a backing store.
auto Data = std::make_pair(std::move(Symbols), std::move(Refs));
return std::make_unique<Dex>(Data.first, Data.second, Rels, std::move(Data),
Size);
Size, SupportContainedRefs);
}

namespace {
Expand Down Expand Up @@ -120,7 +121,7 @@ class IndexBuilder {

} // namespace

void Dex::buildIndex() {
void Dex::buildIndex(bool SupportContainedRefs) {
this->Corpus = dex::Corpus(Symbols.size());
std::vector<std::pair<float, const Symbol *>> ScoredSymbols(Symbols.size());

Expand All @@ -147,6 +148,20 @@ void Dex::buildIndex() {
for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank)
Builder.add(*Symbols[SymbolRank], SymbolRank);
InvertedIndex = std::move(Builder).build();

// If the containedRefs() operation is supported, build the RevRefs
// data structure used to implement it.
if (!SupportContainedRefs)
return;
for (const auto &[ID, RefList] : Refs)
for (const auto &R : RefList)
if ((R.Kind & ContainedRefsRequest::SupportedRefKinds) !=
RefKind::Unknown)
RevRefs.emplace_back(R, ID);
// Sort by container ID so we can use binary search for lookup.
llvm::sort(RevRefs, [](const RevRef &A, const RevRef &B) {
return A.ref().Container < B.ref().Container;
});
}

std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {
Expand Down Expand Up @@ -314,6 +329,36 @@ bool Dex::refs(const RefsRequest &Req,
return false; // We reported all refs.
}

llvm::iterator_range<std::vector<Dex::RevRef>::const_iterator>
Dex::lookupRevRefs(const SymbolID &Container) const {
// equal_range() requires an element of the same type as the elements of the
// range, so construct a dummy RevRef with the container of interest.
Ref QueryRef;
QueryRef.Container = Container;
RevRef Query(QueryRef, SymbolID{});

auto ItPair = std::equal_range(RevRefs.cbegin(), RevRefs.cend(), Query,
[](const RevRef &A, const RevRef &B) {
return A.ref().Container < B.ref().Container;
});
return {ItPair.first, ItPair.second};
}

bool Dex::containedRefs(
const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)> Callback) const {
trace::Span Tracer("Dex reversed refs");
uint32_t Remaining = Req.Limit.value_or(std::numeric_limits<uint32_t>::max());
for (const auto &Rev : lookupRevRefs(Req.ID)) {
// RevRefs are already filtered to ContainedRefsRequest::SupportedRefKinds
if (Remaining == 0)
return true; // More refs were available.
--Remaining;
Callback(Rev.containedRefsResult());
}
return false; // We reported all refs.
}

void Dex::relations(
const RelationsRequest &Req,
llvm::function_ref<void(const SymbolID &, const Symbol &)> Callback) const {
Expand Down Expand Up @@ -350,6 +395,7 @@ size_t Dex::estimateMemoryUsage() const {
for (const auto &TokenToPostingList : InvertedIndex)
Bytes += TokenToPostingList.second.bytes();
Bytes += Refs.getMemorySize();
Bytes += RevRefs.size() * sizeof(RevRef);
Bytes += Relations.getMemorySize();
return Bytes + BackingDataSize;
}
Expand Down
39 changes: 31 additions & 8 deletions clang-tools-extra/clangd/index/dex/Dex.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ class Dex : public SymbolIndex {
public:
// All data must outlive this index.
template <typename SymbolRange, typename RefsRange, typename RelationsRange>
Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations)
Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,
bool SupportContainedRefs)
: Corpus(0) {
for (auto &&Sym : Symbols)
this->Symbols.push_back(&Sym);
Expand All @@ -46,15 +47,15 @@ class Dex : public SymbolIndex {
this->Relations[std::make_pair(Rel.Subject,
static_cast<uint8_t>(Rel.Predicate))]
.push_back(Rel.Object);
buildIndex();
buildIndex(SupportContainedRefs);
}
// Symbols and Refs are owned by BackingData, Index takes ownership.
template <typename SymbolRange, typename RefsRange, typename RelationsRange,
typename Payload>
Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,
Payload &&BackingData, size_t BackingDataSize)
Payload &&BackingData, size_t BackingDataSize, bool SupportContainedRefs)
: Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs),
std::forward<RelationsRange>(Relations)) {
std::forward<RelationsRange>(Relations), SupportContainedRefs) {
KeepAlive = std::shared_ptr<void>(
std::make_shared<Payload>(std::move(BackingData)), nullptr);
this->BackingDataSize = BackingDataSize;
Expand All @@ -64,16 +65,18 @@ class Dex : public SymbolIndex {
typename FileRange, typename Payload>
Dex(SymbolRange &&Symbols, RefsRange &&Refs, RelationsRange &&Relations,
FileRange &&Files, IndexContents IdxContents, Payload &&BackingData,
size_t BackingDataSize)
size_t BackingDataSize, bool SupportContainedRefs)
: Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs),
std::forward<RelationsRange>(Relations),
std::forward<Payload>(BackingData), BackingDataSize) {
std::forward<Payload>(BackingData), BackingDataSize,
SupportContainedRefs) {
this->Files = std::forward<FileRange>(Files);
this->IdxContents = IdxContents;
}

/// Builds an index from slabs. The index takes ownership of the slab.
static std::unique_ptr<SymbolIndex> build(SymbolSlab, RefSlab, RelationSlab);
static std::unique_ptr<SymbolIndex> build(SymbolSlab, RefSlab, RelationSlab,
bool SupportContainedRefs);

bool
fuzzyFind(const FuzzyFindRequest &Req,
Expand All @@ -85,6 +88,10 @@ class Dex : public SymbolIndex {
bool refs(const RefsRequest &Req,
llvm::function_ref<void(const Ref &)> Callback) const override;

bool containedRefs(const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)>
Callback) const override;

void relations(const RelationsRequest &Req,
llvm::function_ref<void(const SymbolID &, const Symbol &)>
Callback) const override;
Expand All @@ -95,7 +102,22 @@ class Dex : public SymbolIndex {
size_t estimateMemoryUsage() const override;

private:
void buildIndex();
class RevRef {
const Ref *Reference;
SymbolID Target;

public:
RevRef(const Ref &Reference, SymbolID Target)
: Reference(&Reference), Target(Target) {}
const Ref &ref() const { return *Reference; }
ContainedRefsResult containedRefsResult() const {
return {ref().Location, ref().Kind, Target};
}
};

void buildIndex(bool EnableOutgoingCalls);
llvm::iterator_range<std::vector<RevRef>::const_iterator>
lookupRevRefs(const SymbolID &Container) const;
std::unique_ptr<Iterator> iterator(const Token &Tok) const;
std::unique_ptr<Iterator>
createFileProximityIterator(llvm::ArrayRef<std::string> ProximityPaths) const;
Expand All @@ -116,6 +138,7 @@ class Dex : public SymbolIndex {
llvm::DenseMap<Token, PostingList> InvertedIndex;
dex::Corpus Corpus;
llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> Refs;
std::vector<RevRef> RevRefs; // sorted by container ID
static_assert(sizeof(RelationKind) == sizeof(uint8_t),
"RelationKind should be of same size as a uint8_t");
llvm::DenseMap<std::pair<SymbolID, uint8_t>, std::vector<SymbolID>> Relations;
Expand Down
3 changes: 2 additions & 1 deletion clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,8 @@ std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
return Index.starts_with("remote:")
? remote::getClient(Index.drop_front(strlen("remote:")),
ProjectRoot)
: loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true);
: loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true,
/*SupportContainedRefs=*/true);
}

bool runCommand(std::string Request, const SymbolIndex &Index) {
Expand Down
7 changes: 7 additions & 0 deletions clang-tools-extra/clangd/index/remote/Client.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,13 @@ class IndexClient : public clangd::SymbolIndex {
return streamRPC(Request, &remote::v1::SymbolIndex::Stub::Refs, Callback);
}

bool containedRefs(const clangd::ContainedRefsRequest &Request,
llvm::function_ref<void(const ContainedRefsResult &)>
Callback) const override {
return streamRPC(Request, &remote::v1::SymbolIndex::Stub::ContainedRefs,
Callback);
}

void
relations(const clangd::RelationsRequest &Request,
llvm::function_ref<void(const SymbolID &, const clangd::Symbol &)>
Expand Down
18 changes: 18 additions & 0 deletions clang-tools-extra/clangd/index/remote/Index.proto
Original file line number Diff line number Diff line change
Expand Up @@ -131,3 +131,21 @@ message Relation {
optional string subject_id = 1;
optional Symbol object = 2;
}

message ContainedRefsRequest {
required string id = 1;
optional uint32 limit = 2;
}

message ContainedRefsReply {
oneof kind {
ContainedRef stream_result = 1;
FinalResult final_result = 2;
}
}

message ContainedRef {
required SymbolLocation location = 1;
required uint32 kind = 2;
required string symbol = 3;
}
2 changes: 2 additions & 0 deletions clang-tools-extra/clangd/index/remote/Service.proto
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,7 @@ service SymbolIndex {

rpc Refs(RefsRequest) returns (stream RefsReply) {}

rpc ContainedRefs(ContainedRefsRequest) returns (stream ContainedRefsReply) {}

rpc Relations(RelationsRequest) returns (stream RelationsReply) {}
}
48 changes: 48 additions & 0 deletions clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,18 @@ Marshaller::fromProtobuf(const RefsRequest *Message) {
return Req;
}

llvm::Expected<clangd::ContainedRefsRequest>
Marshaller::fromProtobuf(const ContainedRefsRequest *Message) {
clangd::ContainedRefsRequest Req;
auto ID = SymbolID::fromStr(Message->id());
if (!ID)
return ID.takeError();
Req.ID = *ID;
if (Message->has_limit())
Req.Limit = Message->limit();
return Req;
}

llvm::Expected<clangd::RelationsRequest>
Marshaller::fromProtobuf(const RelationsRequest *Message) {
clangd::RelationsRequest Req;
Expand Down Expand Up @@ -192,6 +204,21 @@ llvm::Expected<clangd::Ref> Marshaller::fromProtobuf(const Ref &Message) {
return Result;
}

llvm::Expected<clangd::ContainedRefsResult>
Marshaller::fromProtobuf(const ContainedRef &Message) {
clangd::ContainedRefsResult Result;
auto Location = fromProtobuf(Message.location());
if (!Location)
return Location.takeError();
Result.Location = *Location;
Result.Kind = static_cast<RefKind>(Message.kind());
auto Symbol = SymbolID::fromStr(Message.symbol());
if (!Symbol)
return Symbol.takeError();
Result.Symbol = *Symbol;
return Result;
}

llvm::Expected<std::pair<clangd::SymbolID, clangd::Symbol>>
Marshaller::fromProtobuf(const Relation &Message) {
auto SubjectID = SymbolID::fromStr(Message.subject_id());
Expand Down Expand Up @@ -244,6 +271,15 @@ RefsRequest Marshaller::toProtobuf(const clangd::RefsRequest &From) {
return RPCRequest;
}

ContainedRefsRequest
Marshaller::toProtobuf(const clangd::ContainedRefsRequest &From) {
ContainedRefsRequest RPCRequest;
RPCRequest.set_id(From.ID.str());
if (From.Limit)
RPCRequest.set_limit(*From.Limit);
return RPCRequest;
}

RelationsRequest Marshaller::toProtobuf(const clangd::RelationsRequest &From) {
RelationsRequest RPCRequest;
for (const auto &ID : From.Subjects)
Expand Down Expand Up @@ -299,6 +335,18 @@ llvm::Expected<Ref> Marshaller::toProtobuf(const clangd::Ref &From) {
return Result;
}

llvm::Expected<ContainedRef>
Marshaller::toProtobuf(const clangd::ContainedRefsResult &From) {
ContainedRef Result;
auto Location = toProtobuf(From.Location);
if (!Location)
return Location.takeError();
*Result.mutable_location() = *Location;
Result.set_kind(static_cast<uint32_t>(From.Kind));
*Result.mutable_symbol() = From.Symbol.str();
return Result;
}

llvm::Expected<Relation> Marshaller::toProtobuf(const clangd::SymbolID &Subject,
const clangd::Symbol &Object) {
Relation Result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ class Marshaller {

llvm::Expected<clangd::Symbol> fromProtobuf(const Symbol &Message);
llvm::Expected<clangd::Ref> fromProtobuf(const Ref &Message);
llvm::Expected<clangd::ContainedRefsResult>
fromProtobuf(const ContainedRef &Message);
llvm::Expected<std::pair<clangd::SymbolID, clangd::Symbol>>
fromProtobuf(const Relation &Message);

Expand All @@ -48,6 +50,8 @@ class Marshaller {
llvm::Expected<clangd::FuzzyFindRequest>
fromProtobuf(const FuzzyFindRequest *Message);
llvm::Expected<clangd::RefsRequest> fromProtobuf(const RefsRequest *Message);
llvm::Expected<clangd::ContainedRefsRequest>
fromProtobuf(const ContainedRefsRequest *Message);
llvm::Expected<clangd::RelationsRequest>
fromProtobuf(const RelationsRequest *Message);

Expand All @@ -58,10 +62,13 @@ class Marshaller {
LookupRequest toProtobuf(const clangd::LookupRequest &From);
FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From);
RefsRequest toProtobuf(const clangd::RefsRequest &From);
ContainedRefsRequest toProtobuf(const clangd::ContainedRefsRequest &From);
RelationsRequest toProtobuf(const clangd::RelationsRequest &From);

llvm::Expected<Symbol> toProtobuf(const clangd::Symbol &From);
llvm::Expected<Ref> toProtobuf(const clangd::Ref &From);
llvm::Expected<ContainedRef>
toProtobuf(const clangd::ContainedRefsResult &From);
llvm::Expected<Relation> toProtobuf(const clangd::SymbolID &Subject,
const clangd::Symbol &Object);

Expand Down
55 changes: 52 additions & 3 deletions clang-tools-extra/clangd/index/remote/server/Server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,53 @@ class RemoteIndexServer final : public v1::SymbolIndex::Service {
return grpc::Status::OK;
}

grpc::Status
ContainedRefs(grpc::ServerContext *Context,
const ContainedRefsRequest *Request,
grpc::ServerWriter<ContainedRefsReply> *Reply) override {
auto StartTime = stopwatch::now();
WithContextValue WithRequestContext(CurrentRequest, Context);
logRequest(*Request);
trace::Span Tracer("ContainedRefsRequest");
auto Req = ProtobufMarshaller->fromProtobuf(Request);
if (!Req) {
elog("Can not parse ContainedRefsRequest from protobuf: {0}",
Req.takeError());
return grpc::Status::CANCELLED;
}
if (!Req->Limit || *Req->Limit > LimitResults) {
log("[public] Limiting result size for ContainedRefs request from {0} to "
"{1}.",
Req->Limit, LimitResults);
Req->Limit = LimitResults;
}
unsigned Sent = 0;
unsigned FailedToSend = 0;
bool HasMore =
Index.containedRefs(*Req, [&](const clangd::ContainedRefsResult &Item) {
auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
if (!SerializedItem) {
elog("Unable to convert ContainedRefsResult to protobuf: {0}",
SerializedItem.takeError());
++FailedToSend;
return;
}
ContainedRefsReply NextMessage;
*NextMessage.mutable_stream_result() = *SerializedItem;
logResponse(NextMessage);
Reply->Write(NextMessage);
++Sent;
});
ContainedRefsReply LastMessage;
LastMessage.mutable_final_result()->set_has_more(HasMore);
logResponse(LastMessage);
Reply->Write(LastMessage);
SPAN_ATTACH(Tracer, "Sent", Sent);
SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
logRequestSummary("v1/ContainedRefs", Sent, StartTime);
return grpc::Status::OK;
}

grpc::Status Relations(grpc::ServerContext *Context,
const RelationsRequest *Request,
grpc::ServerWriter<RelationsReply> *Reply) override {
Expand Down Expand Up @@ -396,7 +443,8 @@ void hotReload(clangd::SwapIndex &Index, llvm::StringRef IndexPath,
LastStatus.getLastModificationTime(), Status->getLastModificationTime());
LastStatus = *Status;
std::unique_ptr<clang::clangd::SymbolIndex> NewIndex =
loadIndex(IndexPath, SymbolOrigin::Static);
loadIndex(IndexPath, SymbolOrigin::Static, /*UseDex=*/true,
/*SupportContainedRefs=*/true);
if (!NewIndex) {
elog("Failed to load new index. Old index will be served.");
return;
Expand Down Expand Up @@ -532,8 +580,9 @@ int main(int argc, char *argv[]) {
return Status.getError().value();
}

auto SymIndex =
clang::clangd::loadIndex(IndexPath, clang::clangd::SymbolOrigin::Static);
auto SymIndex = clang::clangd::loadIndex(
IndexPath, clang::clangd::SymbolOrigin::Static, /*UseDex=*/true,
/*SupportContainedRefs=*/true);
if (!SymIndex) {
llvm::errs() << "Failed to open the index.\n";
return -1;
Expand Down
Binary file not shown.
2 changes: 2 additions & 0 deletions clang-tools-extra/clangd/test/type-hierarchy-ext.test
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# CHECK-NEXT: "data": {
# CHECK-NEXT: "symbolID": "A6576FE083F2949A"
# CHECK-NEXT: },
# CHECK-NEXT: "detail": "Child3",
# CHECK-NEXT: "kind": 23,
# CHECK-NEXT: "name": "Child3",
# CHECK-NEXT: "range": {
Expand Down Expand Up @@ -153,6 +154,7 @@
# CHECK-NEXT: "data": {
# CHECK-NEXT: "symbolID": "5705B382DFC77CBC"
# CHECK-NEXT: },
# CHECK-NEXT: "detail": "Child4",
# CHECK-NEXT: "kind": 23,
# CHECK-NEXT: "name": "Child4",
# CHECK-NEXT: "range": {
Expand Down
2 changes: 2 additions & 0 deletions clang-tools-extra/clangd/test/type-hierarchy.test
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
# CHECK-NEXT: ],
# CHECK-NEXT: "symbolID": "ECDC0C46D75120F4"
# CHECK-NEXT: },
# CHECK-NEXT: "detail": "Child1",
# CHECK-NEXT: "kind": 23,
# CHECK-NEXT: "name": "Child1",
# CHECK-NEXT: "range": {
Expand Down Expand Up @@ -112,6 +113,7 @@
# CHECK-NEXT: ],
# CHECK-NEXT: "symbolID": "A6576FE083F2949A"
# CHECK-NEXT: },
# CHECK-NEXT: "detail": "Child3",
# CHECK-NEXT: "kind": 23,
# CHECK-NEXT: "name": "Child3",
# CHECK-NEXT: "range": {
Expand Down
2 changes: 1 addition & 1 deletion clang-tools-extra/clangd/tool/Check.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ class Checker {
unsigned ErrCount = 0;

Checker(llvm::StringRef File, const ClangdLSPServer::Options &Opts)
: File(File), Opts(Opts) {}
: File(File), Opts(Opts), Index(/*SupportContainedRefs=*/true) {}

// Read compilation database and choose a compile command for the file.
bool buildCommand(const ThreadsafeFS &TFS) {
Expand Down
14 changes: 10 additions & 4 deletions clang-tools-extra/clangd/tool/ClangdMain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,7 @@ const char TestScheme::TestDir[] = "/clangd-test";

std::unique_ptr<SymbolIndex>
loadExternalIndex(const Config::ExternalIndexSpec &External,
AsyncTaskRunner *Tasks) {
AsyncTaskRunner *Tasks, bool SupportContainedRefs) {
static const trace::Metric RemoteIndexUsed("used_remote_index",
trace::Metric::Value, "address");
switch (External.Kind) {
Expand All @@ -620,8 +620,9 @@ loadExternalIndex(const Config::ExternalIndexSpec &External,
External.Location);
auto NewIndex = std::make_unique<SwapIndex>(std::make_unique<MemIndex>());
auto IndexLoadTask = [File = External.Location,
PlaceHolder = NewIndex.get()] {
if (auto Idx = loadIndex(File, SymbolOrigin::Static, /*UseDex=*/true))
PlaceHolder = NewIndex.get(), SupportContainedRefs] {
if (auto Idx = loadIndex(File, SymbolOrigin::Static, /*UseDex=*/true,
SupportContainedRefs))
PlaceHolder->reset(std::move(Idx));
};
if (Tasks) {
Expand Down Expand Up @@ -909,7 +910,12 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
Opts.BackgroundIndexPriority = BackgroundIndexPriority;
Opts.ReferencesLimit = ReferencesLimit;
Opts.Rename.LimitFiles = RenameFileLimit;
auto PAI = createProjectAwareIndex(loadExternalIndex, Sync);
auto PAI = createProjectAwareIndex(
[SupportContainedRefs = Opts.EnableOutgoingCalls](
const Config::ExternalIndexSpec &External, AsyncTaskRunner *Tasks) {
return loadExternalIndex(External, Tasks, SupportContainedRefs);
},
Sync);
Opts.StaticIndex = PAI.get();
Opts.AsyncThreadsCount = WorkerThreadsCount;
Opts.MemoryCleanup = getMemoryCleanupFunction();
Expand Down
3 changes: 2 additions & 1 deletion clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -685,7 +685,8 @@ TEST_F(BackgroundIndexTest, Reindex) {
class BackgroundIndexRebuilderTest : public testing::Test {
protected:
BackgroundIndexRebuilderTest()
: Source(IndexContents::All), Target(std::make_unique<MemIndex>()),
: Source(IndexContents::All, /*SupportContainedRefs=*/true),
Target(std::make_unique<MemIndex>()),
Rebuilder(&Target, &Source, /*Threads=*/10) {
// Prepare FileSymbols with TestSymbol in it, for checkRebuild.
TestSymbol.ID = SymbolID("foo");
Expand Down
277 changes: 204 additions & 73 deletions clang-tools-extra/clangd/unittests/CallHierarchyTests.cpp

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1703,6 +1703,12 @@ class IndexRequestCollector : public SymbolIndex {
return false;
}

bool containedRefs(
const ContainedRefsRequest &,
llvm::function_ref<void(const ContainedRefsResult &)>) const override {
return false;
}

void relations(const RelationsRequest &,
llvm::function_ref<void(const SymbolID &, const Symbol &)>)
const override {}
Expand Down
46 changes: 24 additions & 22 deletions clang-tools-extra/clangd/unittests/DexTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ TEST(DexSearchTokens, SymbolPath) {

TEST(Dex, Lookup) {
auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
EXPECT_THAT(lookup(*I, SymbolID("ns::abc")), UnorderedElementsAre("ns::abc"));
EXPECT_THAT(lookup(*I, {SymbolID("ns::abc"), SymbolID("ns::xyz")}),
UnorderedElementsAre("ns::abc", "ns::xyz"));
Expand All @@ -489,7 +489,7 @@ TEST(Dex, FuzzyFind) {
auto Index =
Dex::build(generateSymbols({"ns::ABC", "ns::BCD", "::ABC",
"ns::nested::ABC", "other::ABC", "other::A"}),
RefSlab(), RelationSlab());
RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "ABC";
Req.Scopes = {"ns::"};
Expand All @@ -511,7 +511,8 @@ TEST(Dex, FuzzyFind) {
}

TEST(DexTest, DexLimitedNumMatches) {
auto I = Dex::build(generateNumSymbols(0, 100), RefSlab(), RelationSlab());
auto I =
Dex::build(generateNumSymbols(0, 100), RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "5";
Req.AnyScope = true;
Expand All @@ -526,7 +527,7 @@ TEST(DexTest, DexLimitedNumMatches) {
TEST(DexTest, FuzzyMatch) {
auto I = Dex::build(
generateSymbols({"LaughingOutLoud", "LionPopulation", "LittleOldLady"}),
RefSlab(), RelationSlab());
RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "lol";
Req.AnyScope = true;
Expand All @@ -537,7 +538,7 @@ TEST(DexTest, FuzzyMatch) {

TEST(DexTest, ShortQuery) {
auto I = Dex::build(generateSymbols({"_OneTwoFourSix"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
FuzzyFindRequest Req;
Req.AnyScope = true;
bool Incomplete;
Expand Down Expand Up @@ -580,7 +581,7 @@ TEST(DexTest, ShortQuery) {

TEST(DexTest, MatchQualifiedNamesWithoutSpecificScope) {
auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
FuzzyFindRequest Req;
Req.AnyScope = true;
Req.Query = "y";
Expand All @@ -589,7 +590,7 @@ TEST(DexTest, MatchQualifiedNamesWithoutSpecificScope) {

TEST(DexTest, MatchQualifiedNamesWithGlobalScope) {
auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "y";
Req.Scopes = {""};
Expand All @@ -599,7 +600,7 @@ TEST(DexTest, MatchQualifiedNamesWithGlobalScope) {
TEST(DexTest, MatchQualifiedNamesWithOneScope) {
auto I =
Dex::build(generateSymbols({"a::y1", "a::y2", "a::x", "b::y2", "y3"}),
RefSlab(), RelationSlab());
RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "y";
Req.Scopes = {"a::"};
Expand All @@ -609,7 +610,7 @@ TEST(DexTest, MatchQualifiedNamesWithOneScope) {
TEST(DexTest, MatchQualifiedNamesWithMultipleScopes) {
auto I =
Dex::build(generateSymbols({"a::y1", "a::y2", "a::x", "b::y3", "y3"}),
RefSlab(), RelationSlab());
RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "y";
Req.Scopes = {"a::", "b::"};
Expand All @@ -618,7 +619,7 @@ TEST(DexTest, MatchQualifiedNamesWithMultipleScopes) {

TEST(DexTest, NoMatchNestedScopes) {
auto I = Dex::build(generateSymbols({"a::y1", "a::b::y2"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "y";
Req.Scopes = {"a::"};
Expand All @@ -627,7 +628,7 @@ TEST(DexTest, NoMatchNestedScopes) {

TEST(DexTest, WildcardScope) {
auto I = Dex::build(generateSymbols({"a::y1", "a::b::y2", "c::y3"}),
RefSlab(), RelationSlab());
RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.AnyScope = true;
Req.Query = "y";
Expand All @@ -638,7 +639,7 @@ TEST(DexTest, WildcardScope) {

TEST(DexTest, IgnoreCases) {
auto I = Dex::build(generateSymbols({"ns::ABC", "ns::abc"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
FuzzyFindRequest Req;
Req.Query = "AB";
Req.Scopes = {"ns::"};
Expand All @@ -648,15 +649,15 @@ TEST(DexTest, IgnoreCases) {
TEST(DexTest, UnknownPostingList) {
// Regression test: we used to ignore unknown scopes and accept any symbol.
auto I = Dex::build(generateSymbols({"ns::ABC", "ns::abc"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
FuzzyFindRequest Req;
Req.Scopes = {"ns2::"};
EXPECT_THAT(match(*I, Req), UnorderedElementsAre());
}

TEST(DexTest, Lookup) {
auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), RefSlab(),
RelationSlab());
RelationSlab(), true);
EXPECT_THAT(lookup(*I, SymbolID("ns::abc")), UnorderedElementsAre("ns::abc"));
EXPECT_THAT(lookup(*I, {SymbolID("ns::abc"), SymbolID("ns::xyz")}),
UnorderedElementsAre("ns::abc", "ns::xyz"));
Expand All @@ -671,7 +672,7 @@ TEST(DexTest, SymbolIndexOptionsFilter) {
CodeCompletionSymbol.Flags = Symbol::SymbolFlag::IndexedForCodeCompletion;
NonCodeCompletionSymbol.Flags = Symbol::SymbolFlag::None;
std::vector<Symbol> Symbols{CodeCompletionSymbol, NonCodeCompletionSymbol};
Dex I(Symbols, RefSlab(), RelationSlab());
Dex I(Symbols, RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.AnyScope = true;
Req.RestrictForCodeCompletion = false;
Expand All @@ -687,7 +688,7 @@ TEST(DexTest, ProximityPathsBoosting) {
CloseSymbol.CanonicalDeclaration.FileURI = "unittest:///a/b/c/d/e/f/file.h";

std::vector<Symbol> Symbols{CloseSymbol, RootSymbol};
Dex I(Symbols, RefSlab(), RelationSlab());
Dex I(Symbols, RefSlab(), RelationSlab(), true);

FuzzyFindRequest Req;
Req.AnyScope = true;
Expand Down Expand Up @@ -726,15 +727,15 @@ TEST(DexTests, Refs) {
Req.Filter = RefKind::Declaration | RefKind::Definition;

std::vector<std::string> Files;
EXPECT_FALSE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab())
EXPECT_FALSE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab(), true)
.refs(Req, [&](const Ref &R) {
Files.push_back(R.Location.FileURI);
}));
EXPECT_THAT(Files, UnorderedElementsAre("foo.h", "foo.cc"));

Req.Limit = 1;
Files.clear();
EXPECT_TRUE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab())
EXPECT_TRUE(Dex(std::vector<Symbol>{Foo, Bar}, Refs, RelationSlab(), true)
.refs(Req, [&](const Ref &R) {
Files.push_back(R.Location.FileURI);
}));
Expand All @@ -751,7 +752,7 @@ TEST(DexTests, Relations) {
std::vector<Relation> Relations{{Parent.ID, RelationKind::BaseOf, Child1.ID},
{Parent.ID, RelationKind::BaseOf, Child2.ID}};

Dex I{Symbols, RefSlab(), Relations};
Dex I{Symbols, RefSlab(), Relations, true};

std::vector<SymbolID> Results;
RelationsRequest Req;
Expand All @@ -770,7 +771,7 @@ TEST(DexIndex, IndexedFiles) {
auto Data = std::make_pair(std::move(Symbols), std::move(Refs));
llvm::StringSet<> Files = {"unittest:///foo.cc", "unittest:///bar.cc"};
Dex I(std::move(Data.first), std::move(Data.second), RelationSlab(),
std::move(Files), IndexContents::All, std::move(Data), Size);
std::move(Files), IndexContents::All, std::move(Data), Size, true);
auto ContainsFile = I.indexedFiles();
EXPECT_EQ(ContainsFile("unittest:///foo.cc"), IndexContents::All);
EXPECT_EQ(ContainsFile("unittest:///bar.cc"), IndexContents::All);
Expand All @@ -784,7 +785,7 @@ TEST(DexTest, PreferredTypesBoosting) {
Sym2.Type = "T2";

std::vector<Symbol> Symbols{Sym1, Sym2};
Dex I(Symbols, RefSlab(), RelationSlab());
Dex I(Symbols, RefSlab(), RelationSlab(), true);

FuzzyFindRequest Req;
Req.AnyScope = true;
Expand Down Expand Up @@ -820,7 +821,8 @@ TEST(DexTest, TemplateSpecialization) {
index::SymbolProperty::TemplatePartialSpecialization);
B.insert(S);

auto I = dex::Dex::build(std::move(B).build(), RefSlab(), RelationSlab());
auto I =
dex::Dex::build(std::move(B).build(), RefSlab(), RelationSlab(), true);
FuzzyFindRequest Req;
Req.AnyScope = true;

Expand Down
52 changes: 26 additions & 26 deletions clang-tools-extra/clangd/unittests/FileIndexTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ std::unique_ptr<RelationSlab> relSlab(llvm::ArrayRef<const Relation> Rels) {
}

TEST(FileSymbolsTest, UpdateAndGet) {
FileSymbols FS(IndexContents::All);
FileSymbols FS(IndexContents::All, true);
EXPECT_THAT(runFuzzyFind(*FS.buildIndex(IndexType::Light), ""), IsEmpty());

FS.update("f1", numSlab(1, 3), refSlab(SymbolID("1"), "f1.cc"), nullptr,
Expand All @@ -116,7 +116,7 @@ TEST(FileSymbolsTest, UpdateAndGet) {
}

TEST(FileSymbolsTest, Overlap) {
FileSymbols FS(IndexContents::All);
FileSymbols FS(IndexContents::All, true);
FS.update("f1", numSlab(1, 3), nullptr, nullptr, false);
FS.update("f2", numSlab(3, 5), nullptr, nullptr, false);
for (auto Type : {IndexType::Light, IndexType::Heavy})
Expand All @@ -126,7 +126,7 @@ TEST(FileSymbolsTest, Overlap) {
}

TEST(FileSymbolsTest, MergeOverlap) {
FileSymbols FS(IndexContents::All);
FileSymbols FS(IndexContents::All, true);
auto OneSymboSlab = [](Symbol Sym) {
SymbolSlab::Builder S;
S.insert(Sym);
Expand All @@ -147,7 +147,7 @@ TEST(FileSymbolsTest, MergeOverlap) {
}

TEST(FileSymbolsTest, SnapshotAliveAfterRemove) {
FileSymbols FS(IndexContents::All);
FileSymbols FS(IndexContents::All, true);

SymbolID ID("1");
FS.update("f1", numSlab(1, 3), refSlab(ID, "f1.cc"), nullptr, false);
Expand Down Expand Up @@ -180,14 +180,14 @@ void update(FileIndex &M, llvm::StringRef Basename, llvm::StringRef Code) {
}

TEST(FileIndexTest, CustomizedURIScheme) {
FileIndex M;
FileIndex M(true);
update(M, "f", "class string {};");

EXPECT_THAT(runFuzzyFind(M, ""), ElementsAre(declURI("unittest:///f.h")));
}

TEST(FileIndexTest, IndexAST) {
FileIndex M;
FileIndex M(true);
update(M, "f1", "namespace ns { void f() {} class X {}; }");

FuzzyFindRequest Req;
Expand All @@ -198,7 +198,7 @@ TEST(FileIndexTest, IndexAST) {
}

TEST(FileIndexTest, NoLocal) {
FileIndex M;
FileIndex M(true);
update(M, "f1", "namespace ns { void f() { int local = 0; } class X {}; }");

EXPECT_THAT(
Expand All @@ -207,7 +207,7 @@ TEST(FileIndexTest, NoLocal) {
}

TEST(FileIndexTest, IndexMultiASTAndDeduplicate) {
FileIndex M;
FileIndex M(true);
update(M, "f1", "namespace ns { void f() {} class X {}; }");
update(M, "f2", "namespace ns { void ff() {} class X {}; }");

Expand All @@ -219,7 +219,7 @@ TEST(FileIndexTest, IndexMultiASTAndDeduplicate) {
}

TEST(FileIndexTest, ClassMembers) {
FileIndex M;
FileIndex M(true);
update(M, "f1", "class X { static int m1; int m2; static void f(); };");

EXPECT_THAT(runFuzzyFind(M, ""),
Expand All @@ -228,7 +228,7 @@ TEST(FileIndexTest, ClassMembers) {
}

TEST(FileIndexTest, IncludeCollected) {
FileIndex M;
FileIndex M(true);
update(
M, "f",
"// IWYU pragma: private, include <the/good/header.h>\nclass string {};");
Expand All @@ -240,7 +240,7 @@ TEST(FileIndexTest, IncludeCollected) {
}

TEST(FileIndexTest, IWYUPragmaExport) {
FileIndex M;
FileIndex M(true);

TestTU File;
File.Code = R"cpp(#pragma once
Expand Down Expand Up @@ -286,7 +286,7 @@ template <class Ty, class Arg>
vector<Ty> make_vector(Arg A) {}
)cpp";

FileIndex M;
FileIndex M(true);
update(M, "f", Source);

auto Symbols = runFuzzyFind(M, "");
Expand Down Expand Up @@ -334,7 +334,7 @@ TEST(FileIndexTest, RebuildWithPreamble) {
IgnoreDiagnostics IgnoreDiags;
auto CI = buildCompilerInvocation(PI, IgnoreDiags);

FileIndex Index;
FileIndex Index(true);
bool IndexUpdated = false;
buildPreamble(
FooCpp, *CI, PI,
Expand Down Expand Up @@ -374,7 +374,7 @@ TEST(FileIndexTest, Refs) {
RefsRequest Request;
Request.IDs = {Foo.ID};

FileIndex Index;
FileIndex Index(true);
// Add test.cc
TestTU Test;
Test.HeaderCode = HeaderCode;
Expand Down Expand Up @@ -409,7 +409,7 @@ TEST(FileIndexTest, MacroRefs) {
}
)cpp");

FileIndex Index;
FileIndex Index(true);
// Add test.cc
TestTU Test;
Test.HeaderCode = std::string(HeaderCode.code());
Expand All @@ -432,7 +432,7 @@ TEST(FileIndexTest, MacroRefs) {
}

TEST(FileIndexTest, CollectMacros) {
FileIndex M;
FileIndex M(true);
update(M, "f", "#define CLANGD 1");
EXPECT_THAT(runFuzzyFind(M, ""), Contains(qName("CLANGD")));
}
Expand All @@ -443,7 +443,7 @@ TEST(FileIndexTest, Relations) {
TU.HeaderFilename = "f.h";
TU.HeaderCode = "class A {}; class B : public A {};";
auto AST = TU.build();
FileIndex Index;
FileIndex Index(true);
Index.updatePreamble(testPath(TU.Filename), /*Version=*/"null",
AST.getASTContext(), AST.getPreprocessor(),
AST.getPragmaIncludes());
Expand Down Expand Up @@ -493,7 +493,7 @@ TEST(FileIndexTest, ReferencesInMainFileWithPreamble) {
)cpp");
TU.Code = std::string(Main.code());
auto AST = TU.build();
FileIndex Index;
FileIndex Index(true);
Index.updateMain(testPath(TU.Filename), AST);

// Expect to see references in main file, references in headers are excluded
Expand All @@ -510,7 +510,7 @@ TEST(FileIndexTest, MergeMainFileSymbols) {
Cpp.HeaderFilename = "foo.h";
Cpp.HeaderCode = CommonHeader;

FileIndex Index;
FileIndex Index(true);
auto HeaderAST = Header.build();
auto CppAST = Cpp.build();
Index.updateMain(testPath("foo.h"), HeaderAST);
Expand All @@ -524,7 +524,7 @@ TEST(FileIndexTest, MergeMainFileSymbols) {
}

TEST(FileSymbolsTest, CountReferencesNoRefSlabs) {
FileSymbols FS(IndexContents::All);
FileSymbols FS(IndexContents::All, true);
FS.update("f1", numSlab(1, 3), nullptr, nullptr, true);
FS.update("f2", numSlab(1, 3), nullptr, nullptr, false);
EXPECT_THAT(
Expand All @@ -536,7 +536,7 @@ TEST(FileSymbolsTest, CountReferencesNoRefSlabs) {
}

TEST(FileSymbolsTest, CountReferencesWithRefSlabs) {
FileSymbols FS(IndexContents::All);
FileSymbols FS(IndexContents::All, true);
FS.update("f1cpp", numSlab(1, 3), refSlab(SymbolID("1"), "f1.cpp"), nullptr,
true);
FS.update("f1h", numSlab(1, 3), refSlab(SymbolID("1"), "f1.h"), nullptr,
Expand All @@ -558,7 +558,7 @@ TEST(FileSymbolsTest, CountReferencesWithRefSlabs) {
}

TEST(FileIndexTest, StalePreambleSymbolsDeleted) {
FileIndex M;
FileIndex M(true);
TestTU File;
File.HeaderFilename = "a.h";

Expand All @@ -581,7 +581,7 @@ TEST(FileIndexTest, StalePreambleSymbolsDeleted) {

// Verifies that concurrent calls to updateMain don't "lose" any updates.
TEST(FileIndexTest, Threadsafety) {
FileIndex M;
FileIndex M(true);
Notification Go;

constexpr int Count = 10;
Expand Down Expand Up @@ -714,7 +714,7 @@ TEST(FileShardedIndexTest, Sharding) {
}

TEST(FileIndexTest, Profile) {
FileIndex FI;
FileIndex FI(true);

auto FileName = testPath("foo.cpp");
auto AST = TestTU::withHeaderCode("int a;").build();
Expand All @@ -738,7 +738,7 @@ TEST(FileIndexTest, Profile) {
}

TEST(FileSymbolsTest, Profile) {
FileSymbols FS(IndexContents::All);
FileSymbols FS(IndexContents::All, true);
FS.update("f1", numSlab(1, 2), nullptr, nullptr, false);
FS.update("f2", nullptr, refSlab(SymbolID("1"), "f1"), nullptr, false);
FS.update("f3", nullptr, nullptr,
Expand All @@ -758,7 +758,7 @@ TEST(FileSymbolsTest, Profile) {
}

TEST(FileIndexTest, MacrosFromMainFile) {
FileIndex Idx;
FileIndex Idx(true);
TestTU TU;
TU.Code = "#pragma once\n#define FOO";
TU.Filename = "foo.h";
Expand Down
8 changes: 4 additions & 4 deletions clang-tools-extra/clangd/unittests/IndexTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ TEST(MergeIndexTest, Lookup) {
}

TEST(MergeIndexTest, LookupRemovedDefinition) {
FileIndex DynamicIndex, StaticIndex;
FileIndex DynamicIndex(true), StaticIndex(true);
MergedIndex Merge(&DynamicIndex, &StaticIndex);

const char *HeaderCode = "class Foo;";
Expand Down Expand Up @@ -349,7 +349,7 @@ TEST(MergeIndexTest, FuzzyFind) {
}

TEST(MergeIndexTest, FuzzyFindRemovedSymbol) {
FileIndex DynamicIndex, StaticIndex;
FileIndex DynamicIndex(true), StaticIndex(true);
MergedIndex Merge(&DynamicIndex, &StaticIndex);

const char *HeaderCode = "class Foo;";
Expand Down Expand Up @@ -446,8 +446,8 @@ TEST(MergeTest, PreferSymbolLocationInCodegenFile) {
}

TEST(MergeIndexTest, Refs) {
FileIndex Dyn;
FileIndex StaticIndex;
FileIndex Dyn(true);
FileIndex StaticIndex(true);
MergedIndex Merge(&Dyn, &StaticIndex);

const char *HeaderCode = "class Foo;";
Expand Down
14 changes: 13 additions & 1 deletion clang-tools-extra/clangd/unittests/RenameTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1548,7 +1548,7 @@ TEST(CrossFileRenameTests, DirtyBuffer) {
std::string BarPath = testPath("bar.cc");
// Build the index, the index has "Foo" references from foo.cc and "Bar"
// references from bar.cc.
FileSymbols FSymbols(IndexContents::All);
FileSymbols FSymbols(IndexContents::All, true);
FSymbols.update(FooPath, nullptr, buildRefSlab(FooCode, "Foo", FooPath),
nullptr, false);
FSymbols.update(BarPath, nullptr, buildRefSlab(BarCode, "Bar", BarPath),
Expand Down Expand Up @@ -1601,6 +1601,12 @@ TEST(CrossFileRenameTests, DirtyBuffer) {
return true; // has more references
}

bool containedRefs(const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)>
Callback) const override {
return false;
}

bool fuzzyFind(
const FuzzyFindRequest &Req,
llvm::function_ref<void(const Symbol &)> Callback) const override {
Expand Down Expand Up @@ -1652,6 +1658,12 @@ TEST(CrossFileRenameTests, DeduplicateRefsFromIndex) {
return false;
}

bool containedRefs(const ContainedRefsRequest &Req,
llvm::function_ref<void(const ContainedRefsResult &)>
Callback) const override {
return false;
}

bool fuzzyFind(const FuzzyFindRequest &,
llvm::function_ref<void(const Symbol &)>) const override {
return false;
Expand Down
2 changes: 1 addition & 1 deletion clang-tools-extra/clangd/unittests/TestTU.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ RefSlab TestTU::headerRefs() const {

std::unique_ptr<SymbolIndex> TestTU::index() const {
auto AST = build();
auto Idx = std::make_unique<FileIndex>();
auto Idx = std::make_unique<FileIndex>(/*SupportContainedRefs=*/true);
Idx->updatePreamble(testPath(Filename), /*Version=*/"null",
AST.getASTContext(), AST.getPreprocessor(),
AST.getPragmaIncludes());
Expand Down
2 changes: 1 addition & 1 deletion clang-tools-extra/clangd/unittests/TestWorkspace.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ namespace clang {
namespace clangd {

std::unique_ptr<SymbolIndex> TestWorkspace::index() {
auto Index = std::make_unique<FileIndex>();
auto Index = std::make_unique<FileIndex>(/*SupportContainedRefs=*/true);
for (const auto &Input : Inputs) {
if (!Input.second.IsMainFile)
continue;
Expand Down
6 changes: 4 additions & 2 deletions clang-tools-extra/docs/ReleaseNotes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ Changes in existing checks
<clang-tidy/checks/bugprone/return-const-ref-from-parameter>` check to
diagnose potential dangling references when returning a ``const &`` parameter
by using the conditional operator ``cond ? var1 : var2`` and no longer giving
false positives for functions which contain lambda.
false positives for functions which contain lambda and ignore parameters
with ``[[clang::lifetimebound]]`` attribute.

- Improved :doc:`bugprone-sizeof-expression
<clang-tidy/checks/bugprone/sizeof-expression>` check to find suspicious
Expand Down Expand Up @@ -248,7 +249,8 @@ Changes in existing checks
<clang-tidy/checks/misc/use-internal-linkage>` check to insert ``static``
keyword before type qualifiers such as ``const`` and ``volatile`` and fix
false positives for function declaration without body and fix false positives
for global scoped overloaded ``operator new`` and ``operator delete``.
for C++20 export declarations and fix false positives for global scoped
overloaded ``operator new`` and ``operator delete``.

- Improved :doc:`modernize-avoid-c-arrays
<clang-tidy/checks/modernize/avoid-c-arrays>` check to suggest using
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,6 @@ after the call. When the function returns such a parameter also as constant refe
then the returned reference can be used after the object it refers to has been
destroyed.

This issue can be resolved by declaring an overload of the problematic function
where the ``const &`` parameter is instead declared as ``&&``. The developer has
to ensure that the implementation of that function does not produce a
use-after-free, the exact error that this check is warning against.
Marking such an ``&&`` overload as ``deleted``, will silence the warning as
well. In the case of different ``const &`` parameters being returned depending
on the control flow of the function, an overload where all problematic
``const &`` parameters have been declared as ``&&`` will resolve the issue.

Example
-------

Expand All @@ -38,3 +29,23 @@ Example

const S& s = fn(S{1});
s.v; // use after free


This issue can be resolved by declaring an overload of the problematic function
where the ``const &`` parameter is instead declared as ``&&``. The developer has
to ensure that the implementation of that function does not produce a
use-after-free, the exact error that this check is warning against.
Marking such an ``&&`` overload as ``deleted``, will silence the warning as
well. In the case of different ``const &`` parameters being returned depending
on the control flow of the function, an overload where all problematic
``const &`` parameters have been declared as ``&&`` will resolve the issue.

This issue can also be resolved by adding ``[[clang::lifetimebound]]``. Clang
enable ``-Wdangling`` warning by default which can detect mis-uses of the
annotated function. See `lifetimebound attribute <https://clang.llvm.org/docs/AttributeReference.html#id11>`_
for details.

.. code-block:: c++

const int &f(const int &a [[clang::lifetimebound]]) { return a; } // no warning
const int &v = f(1); // warning: temporary bound to local reference 'v' will be destroyed at the end of the full-expression [-Wdangling]
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ Example:
void fn3(); // without function body in all declaration, maybe external linkage
void fn3();

// export declarations
export void fn4() {}
export namespace t { void fn5() {} }
export int v2;

Options
-------

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@ Checks whether a call to the ``size()``/``length()`` method can be replaced
with a call to ``empty()``.

The emptiness of a container should be checked using the ``empty()`` method
instead of the ``size()``/``length()`` method. It is not guaranteed that
``size()``/``length()`` is a constant-time function, and it is generally more
efficient and also shows clearer intent to use ``empty()``. Furthermore some
containers may implement the ``empty()`` method but not implement the ``size()``
or ``length()`` method. Using ``empty()`` whenever possible makes it easier to
switch to another container in the future.
instead of the ``size()``/``length()`` method. It shows clearer intent to use
``empty()``. Furthermore some containers may implement the ``empty()`` method
but not implement the ``size()`` or ``length()`` method. Using ``empty()``
whenever possible makes it easier to switch to another container in the future.

The check issues warning if a container has ``empty()`` and ``size()`` or
``length()`` methods matching following signatures:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,3 +197,9 @@ int const &overload_params_difference3(int p1, int const &a, int p2) { return a;
int const &overload_params_difference3(int p1, long &&a, int p2);

} // namespace overload

namespace gh117696 {
namespace use_lifetime_bound_attr {
int const &f(int const &a [[clang::lifetimebound]]) { return a; }
} // namespace use_lifetime_bound_attr
} // namespace gh117696
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// RUN: %check_clang_tidy -std=c++20 %s misc-use-internal-linkage %t -- -- -I%S/Inputs/use-internal-linkage

module;

export module test;

export void single_export_fn() {}
export int single_export_var;

export {
void group_export_fn1() {}
void group_export_fn2() {}
int group_export_var1;
int group_export_var2;
}

export namespace aa {
void namespace_export_fn() {}
int namespace_export_var;
} // namespace aa
2 changes: 1 addition & 1 deletion clang/docs/ClangFormat.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ to format C/C++/Java/JavaScript/JSON/Objective-C/Protobuf/C# code.
supported:
CSharp: .cs
Java: .java
JavaScript: .mjs .js .ts
JavaScript: .js .mjs .cjs .ts
Json: .json
Objective-C: .m .mm
Proto: .proto .protodevel
Expand Down
12 changes: 10 additions & 2 deletions clang/docs/LanguageExtensions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -648,7 +648,7 @@ elementwise to the input.
Unless specified otherwise operation(±0) = ±0 and operation(±infinity) = ±infinity

The integer elementwise intrinsics, including ``__builtin_elementwise_popcount``,
can be called in a ``constexpr`` context.
``__builtin_elementwise_bitreverse``, can be called in a ``constexpr`` context.

============================================== ====================================================================== =========================================
Name Operation Supported element types
Expand Down Expand Up @@ -1989,7 +1989,7 @@ Enumerations with a fixed underlying type
-----------------------------------------

Clang provides support for C++11 enumerations with a fixed underlying type
within Objective-C. For example, one can write an enumeration type as:
within Objective-C and C `prior to C23 <https://open-std.org/JTC1/SC22/WG14/www/docs/n3030.htm>`_. For example, one can write an enumeration type as:

.. code-block:: c++

Expand All @@ -2001,6 +2001,14 @@ value, is ``unsigned char``.
Use ``__has_feature(objc_fixed_enum)`` to determine whether support for fixed
underlying types is available in Objective-C.

Use ``__has_extension(c_fixed_enum)`` to determine whether support for fixed
underlying types is available in C prior to C23. This will also report ``true`` in C23
and later modes as the functionality is available even if it's not an extension in
those modes.

Use ``__has_feature(c_fixed_enum)`` to determine whether support for fixed
underlying types is available in C23 and later.

Interoperability with C++11 lambdas
-----------------------------------

Expand Down
11 changes: 11 additions & 0 deletions clang/docs/LibASTMatchersReference.html
Original file line number Diff line number Diff line change
Expand Up @@ -804,6 +804,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
</pre></td></tr>


<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('exportDecl0')"><a name="exportDecl0Anchor">exportDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExportDecl.html">ExportDecl</a>&gt;...</td></tr>
<tr><td colspan="4" class="doc" id="exportDecl0"><pre>Matches any export declaration.

Example matches following declarations.
export void foo();
export { void foo(); }
export namespace { void foo(); }
export int v;
</pre></td></tr>


<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('fieldDecl0')"><a name="fieldDecl0Anchor">fieldDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FieldDecl.html">FieldDecl</a>&gt;...</td></tr>
<tr><td colspan="4" class="doc" id="fieldDecl0"><pre>Matches field declarations.

Expand Down
27 changes: 24 additions & 3 deletions clang/docs/ReleaseNotes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,8 @@ C++2c Feature Support
- Added the ``__builtin_is_within_lifetime`` builtin, which supports
`P2641R4 Checking if a union alternative is active <https://wg21.link/p2641r4>`_

- Implemented `P3176R1 The Oxford variadic comma <https://wg21.link/P3176R1>`_

C++23 Feature Support
^^^^^^^^^^^^^^^^^^^^^
- Removed the restriction to literal types in constexpr functions in C++23 mode.
Expand Down Expand Up @@ -308,6 +310,9 @@ Resolutions to C++ Defect Reports
by default.
(`CWG2521: User-defined literals and reserved identifiers <https://cplusplus.github.io/CWG/issues/2521.html>`_).

- Fix name lookup for a dependent base class that is the current instantiation.
(`CWG591: When a dependent base class is the current instantiation <https://cplusplus.github.io/CWG/issues/591.html>`_).

C Language Changes
------------------

Expand Down Expand Up @@ -402,6 +407,7 @@ Non-comprehensive list of changes in this release
- ``__builtin_reduce_and`` function can now be used in constant expressions.
- ``__builtin_reduce_or`` and ``__builtin_reduce_xor`` functions can now be used in constant expressions.
- ``__builtin_elementwise_popcount`` function can now be used in constant expressions.
- ``__builtin_elementwise_bitreverse`` function can now be used in constant expressions.

New Compiler Flags
------------------
Expand Down Expand Up @@ -624,9 +630,6 @@ Improvements to Clang's diagnostics

- Fixed a false negative ``-Wunused-private-field`` diagnostic when a defaulted comparison operator is defined out of class (#GH116961).

- Clang now supports using alias templates in deduction guides, aligning with the C++ standard,
which treats alias templates as synonyms for their underlying types (#GH54909).

- Clang now diagnoses dangling references for C++20's parenthesized aggregate initialization (#101957).

Improvements to Clang's time-trace
Expand Down Expand Up @@ -758,11 +761,15 @@ Bug Fixes to C++ Support
- Name independent data members were not correctly initialized from default member initializers. (#GH114069)
- Fixed expression transformation for ``[[assume(...)]]``, allowing using pack indexing expressions within the
assumption if they also occur inside of a dependent lambda. (#GH114787)
- Lambdas now capture function types without considering top-level const qualifiers. (#GH84961)
- Clang now uses valid deduced type locations when diagnosing functions with trailing return type
missing placeholder return type. (#GH78694)
- Fixed a bug where bounds of partially expanded pack indexing expressions were checked too early. (#GH116105)
- Fixed an assertion failure caused by using ``consteval`` in condition in consumed analyses. (#GH117385)
- Fix a crash caused by incorrect argument position in merging deduced template arguments. (#GH113659)
- Fixed an assertion failure caused by mangled names with invalid identifiers. (#GH112205)
- Fixed an incorrect lambda scope of generic lambdas that caused Clang to crash when computing potential lambda
captures at the end of a full expression. (#GH115931)

Bug Fixes to AST Handling
^^^^^^^^^^^^^^^^^^^^^^^^^
Expand Down Expand Up @@ -929,9 +936,15 @@ and `-mbulk-memory` flags, which correspond to the [Bulk Memory Operations]
and [Non-trapping float-to-int Conversions] language features, which are
[widely implemented in engines].

A new Lime1 target CPU is added, -mcpu=lime1. This CPU follows the definition of
the Lime1 CPU [here], and enables -mmultivalue, -mmutable-globals,
-mcall-indirect-overlong, -msign-ext, -mbulk-memory-opt, -mnontrapping-fptoint,
and -mextended-const.

[Bulk Memory Operations]: https://github.com/WebAssembly/bulk-memory-operations/blob/master/proposals/bulk-memory-operations/Overview.md
[Non-trapping float-to-int Conversions]: https://github.com/WebAssembly/spec/blob/master/proposals/nontrapping-float-to-int-conversion/Overview.md
[widely implemented in engines]: https://webassembly.org/features/
[here]: https://github.com/WebAssembly/tool-conventions/blob/main/Lime.md#lime1

AVR Support
^^^^^^^^^^^
Expand Down Expand Up @@ -965,6 +978,14 @@ AST Matchers
- Ensure ``hasName`` matches template specializations across inline namespaces,
making `matchesNodeFullSlow` and `matchesNodeFullFast` consistent.

- Improved the performance of the ``getExpansionLocOfMacro`` by tracking already processed macros during recursion.

- Add ``exportDecl`` matcher to match export declaration.

- Ensure ``hasType`` and ``hasDeclaration`` match Objective-C interface declarations.

- Ensure ``pointee`` matches Objective-C pointer types.

clang-format
------------

Expand Down
9 changes: 7 additions & 2 deletions clang/docs/tools/dump_ast_matchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import collections
import re
import os

try:
from urllib.request import urlopen
Expand All @@ -18,7 +19,11 @@
CLASS_INDEX_PAGE = None
print("Unable to get %s: %s" % (CLASS_INDEX_PAGE_URL, e))

MATCHERS_FILE = "../../include/clang/ASTMatchers/ASTMatchers.h"
CURRENT_DIR = os.path.dirname(__file__)
MATCHERS_FILE = os.path.join(
CURRENT_DIR, "../../include/clang/ASTMatchers/ASTMatchers.h"
)
HTML_FILE = os.path.join(CURRENT_DIR, "../LibASTMatchersReference.html")

# Each matcher is documented in one row of the form:
# result | name | argA
Expand Down Expand Up @@ -590,7 +595,7 @@ def sort_table(matcher_type, matcher_map):
narrowing_matcher_table = sort_table("NARROWING", narrowing_matchers)
traversal_matcher_table = sort_table("TRAVERSAL", traversal_matchers)

reference = open("../LibASTMatchersReference.html").read()
reference = open(HTML_FILE).read()
reference = re.sub(
r"<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->",
node_matcher_table,
Expand Down
2 changes: 2 additions & 0 deletions clang/include/clang/AST/Type.h
Original file line number Diff line number Diff line change
Expand Up @@ -3754,6 +3754,8 @@ class ArrayParameterType : public ConstantArrayType {
static bool classof(const Type *T) {
return T->getTypeClass() == ArrayParameter;
}

QualType getConstantArrayType(const ASTContext &Ctx) const;
};

/// Represents a C array with an unspecified size. For example 'int A[]' has
Expand Down
16 changes: 14 additions & 2 deletions clang/include/clang/ASTMatchers/ASTMatchers.h
Original file line number Diff line number Diff line change
Expand Up @@ -1236,6 +1236,17 @@ AST_MATCHER_P(TemplateArgument, equalsIntegralValue,
extern const internal::VariadicDynCastAllOfMatcher<Stmt,
ObjCAutoreleasePoolStmt> autoreleasePoolStmt;

/// Matches any export declaration.
///
/// Example matches following declarations.
/// \code
/// export void foo();
/// export { void foo(); }
/// export namespace { void foo(); }
/// export int v;
/// \endcode
extern const internal::VariadicDynCastAllOfMatcher<Decl, ExportDecl> exportDecl;

/// Matches any value declaration.
///
/// Example matches A, B, C and F
Expand Down Expand Up @@ -4033,7 +4044,7 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
hasType,
AST_POLYMORPHIC_SUPPORTED_TYPES(Expr, FriendDecl, ValueDecl,
CXXBaseSpecifier),
CXXBaseSpecifier, ObjCInterfaceDecl),
internal::Matcher<Decl>, InnerMatcher, 1) {
QualType QT = internal::getUnderlyingType(Node);
if (!QT.isNull())
Expand Down Expand Up @@ -7434,7 +7445,8 @@ extern const AstTypeMatcher<RValueReferenceType> rValueReferenceType;
AST_TYPELOC_TRAVERSE_MATCHER_DECL(
pointee, getPointee,
AST_POLYMORPHIC_SUPPORTED_TYPES(BlockPointerType, MemberPointerType,
PointerType, ReferenceType));
PointerType, ReferenceType,
ObjCObjectPointerType));

/// Matches typedef types.
///
Expand Down
10 changes: 9 additions & 1 deletion clang/include/clang/ASTMatchers/ASTMatchersInternal.h
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,9 @@ inline QualType getUnderlyingType(const FriendDecl &Node) {
inline QualType getUnderlyingType(const CXXBaseSpecifier &Node) {
return Node.getType();
}
inline QualType getUnderlyingType(const ObjCInterfaceDecl &Node) {
return Node.getTypeForDecl()->getPointeeType();
}

/// Unifies obtaining a `TypeSourceInfo` from different node types.
template <typename T,
Expand Down Expand Up @@ -1113,6 +1116,11 @@ class HasDeclarationMatcher : public MatcherInterface<T> {
return matchesDecl(Node.getDecl(), Finder, Builder);
}

bool matchesSpecialized(const ObjCInterfaceDecl &Node, ASTMatchFinder *Finder,
BoundNodesTreeBuilder *Builder) const {
return matchesDecl(Node.getCanonicalDecl(), Finder, Builder);
}

/// Extracts the operator new of the new call and returns whether the
/// inner matcher matches on it.
bool matchesSpecialized(const CXXNewExpr &Node,
Expand Down Expand Up @@ -1213,7 +1221,7 @@ using HasDeclarationSupportedTypes =
ElaboratedType, InjectedClassNameType, LabelStmt, AddrLabelExpr,
MemberExpr, QualType, RecordType, TagType,
TemplateSpecializationType, TemplateTypeParmType, TypedefType,
UnresolvedUsingType, ObjCIvarRefExpr>;
UnresolvedUsingType, ObjCIvarRefExpr, ObjCInterfaceDecl>;

/// A Matcher that allows binding the node it matches to an id.
///
Expand Down
14 changes: 13 additions & 1 deletion clang/include/clang/Basic/Builtins.td
Original file line number Diff line number Diff line change
Expand Up @@ -1270,7 +1270,7 @@ def ElementwiseATan2 : Builtin {

def ElementwiseBitreverse : Builtin {
let Spellings = ["__builtin_elementwise_bitreverse"];
let Attributes = [NoThrow, Const, CustomTypeChecking];
let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
let Prototype = "void(...)";
}

Expand Down Expand Up @@ -4738,6 +4738,12 @@ def GetDeviceSideMangledName : LangBuiltin<"CUDA_LANG"> {
}

// HLSL
def HLSLResourceGetPointer : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_resource_getpointer"];
let Attributes = [NoThrow];
let Prototype = "void(...)";
}

def HLSLAll : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_all"];
let Attributes = [NoThrow, Const];
Expand Down Expand Up @@ -4924,6 +4930,12 @@ def HLSLClip: LangBuiltin<"HLSL_LANG"> {
let Prototype = "void(...)";
}

def HLSLGroupMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_group_memory_barrier_with_group_sync"];
let Attributes = [NoThrow, Const];
let Prototype = "void()";
}

// Builtins for XRay.
def XRayCustomEvent : Builtin {
let Spellings = ["__xray_customevent"];
Expand Down
2 changes: 2 additions & 0 deletions clang/include/clang/Basic/BuiltinsAMDGPU.def
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,8 @@ TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr4_b64_v2i32, "V2iV2i*3", "nc", "gfx950
TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr6_b96_v3i32, "V3iV3i*3", "nc", "gfx950-insts")
TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr8_b64_v2i32, "V2iV2i*3", "nc", "gfx950-insts")
TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr16_b64_v4i16, "V4sV4s*3", "nc", "gfx950-insts")
TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr16_b64_v4f16, "V4hV4h*3", "nc", "gfx950-insts")
TARGET_BUILTIN(__builtin_amdgcn_ds_read_tr16_b64_v4bf16, "V4yV4y*3", "nc", "gfx950-insts")

TARGET_BUILTIN(__builtin_amdgcn_ashr_pk_i8_i32, "UsUiUiUi", "nc", "ashr-pk-insts")
TARGET_BUILTIN(__builtin_amdgcn_ashr_pk_u8_i32, "UsUiUiUi", "nc", "ashr-pk-insts")
Expand Down
14 changes: 9 additions & 5 deletions clang/include/clang/Basic/DiagnosticGroups.td
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ def DeprecatedWritableStr : DiagGroup<"deprecated-writable-strings",
[CXX11CompatDeprecatedWritableStr]>;
def DeprecatedPragma : DiagGroup<"deprecated-pragma">;
def DeprecatedType : DiagGroup<"deprecated-type">;
def DeprecatedMissingCommaVariadicParam : DiagGroup<"deprecated-missing-comma-variadic-parameter">;
// FIXME: Why is DeprecatedImplementations not in this group?
def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
DeprecatedArrayCompare,
Expand All @@ -235,7 +236,8 @@ def Deprecated : DiagGroup<"deprecated", [DeprecatedAnonEnumEnumConversion,
DeprecatedType,
DeprecatedVolatile,
DeprecatedWritableStr,
DeprecatedRedundantConstexprStaticDef
DeprecatedRedundantConstexprStaticDef,
DeprecatedMissingCommaVariadicParam
]>,
DiagCategory<"Deprecations">;

Expand Down Expand Up @@ -292,11 +294,13 @@ def : DiagGroup<"c++1z-compat-mangling", [CXX17CompatMangling]>;
// Name of this warning in GCC.
def NoexceptType : DiagGroup<"noexcept-type", [CXX17CompatMangling]>;

def VariadicMacroArgumentsOmitted : DiagGroup<"variadic-macro-arguments-omitted">;

// Warnings for C code which is not compatible with previous C standards.
def CPre11Compat : DiagGroup<"pre-c11-compat">;
def CPre11CompatPedantic : DiagGroup<"pre-c11-compat-pedantic",
[CPre11Compat]>;
def CPre23Compat : DiagGroup<"pre-c23-compat">;
def CPre23Compat : DiagGroup<"pre-c23-compat", [VariadicMacroArgumentsOmitted]>;
def CPre23CompatPedantic : DiagGroup<"pre-c23-compat-pedantic",
[CPre23Compat]>;
def : DiagGroup<"pre-c2x-compat", [CPre23Compat]>;
Expand Down Expand Up @@ -904,7 +908,7 @@ def VolatileRegisterVar : DiagGroup<"volatile-register-var">;
def Visibility : DiagGroup<"visibility">;
def ZeroLengthArray : DiagGroup<"zero-length-array">;
def GNUZeroLineDirective : DiagGroup<"gnu-zero-line-directive">;
def GNUZeroVariadicMacroArguments : DiagGroup<"gnu-zero-variadic-macro-arguments">;
def GNUZeroVariadicMacroArguments : DiagGroup<"gnu-zero-variadic-macro-arguments", [VariadicMacroArgumentsOmitted]>;
def MisleadingIndentation : DiagGroup<"misleading-indentation">;
def PtrAuthNullPointers : DiagGroup<"ptrauth-null-pointers">;

Expand Down Expand Up @@ -1197,7 +1201,7 @@ def CXX17 : DiagGroup<"c++17-extensions", [CXX17Attrs]>;

// A warning group for warnings about using C++20 features as extensions in
// earlier C++ versions.
def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs]>;
def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs, VariadicMacroArgumentsOmitted]>;

// A warning group for warnings about using C++23 features as extensions in
// earlier C++ versions.
Expand All @@ -1224,7 +1228,7 @@ def C11 : DiagGroup<"c11-extensions">;
def C99 : DiagGroup<"c99-extensions", [C99Designator]>;

// A warning group for warnings about using C23 features as extensions.
def C23 : DiagGroup<"c23-extensions">;
def C23 : DiagGroup<"c23-extensions", [VariadicMacroArgumentsOmitted]>;

def : DiagGroup<"c2x-extensions", [C23]>;

Expand Down
6 changes: 3 additions & 3 deletions clang/include/clang/Basic/DiagnosticLexKinds.td
Original file line number Diff line number Diff line change
Expand Up @@ -486,14 +486,14 @@ def ext_embedded_directive : Extension<
InGroup<DiagGroup<"embedded-directive">>;
def ext_c_missing_varargs_arg : Extension<
"passing no argument for the '...' parameter of a variadic macro is "
"a C23 extension">, InGroup<C23>;
"a C23 extension">, InGroup<VariadicMacroArgumentsOmitted>;
def ext_cxx_missing_varargs_arg : Extension<
"passing no argument for the '...' parameter of a variadic macro is "
"a C++20 extension">, InGroup<CXX20>;
"a C++20 extension">, InGroup<VariadicMacroArgumentsOmitted>;
def warn_c17_compat_missing_varargs_arg : Warning<
"passing no argument for the '...' parameter of a variadic macro is "
"incompatible with C standards before C23">,
InGroup<CPre23Compat>, DefaultIgnore;
InGroup<VariadicMacroArgumentsOmitted>, DefaultIgnore;
def warn_cxx17_compat_missing_varargs_arg : Warning<
"passing no argument for the '...' parameter of a variadic macro is "
"incompatible with C++ standards before C++20">,
Expand Down
3 changes: 3 additions & 0 deletions clang/include/clang/Basic/DiagnosticParseKinds.td
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,9 @@ def err_function_scope_depth_exceeded : Error<
"function scope depth exceeded maximum of %0">, DefaultFatal;
def err_missing_comma_before_ellipsis : Error<
"C requires a comma prior to the ellipsis in a variadic function type">;
def warn_deprecated_missing_comma_before_ellipsis : Warning<
"declaration of a variadic function without a comma before '...' is deprecated">,
InGroup<DeprecatedMissingCommaVariadicParam>;
def err_unexpected_typedef_ident : Error<
"unexpected type name %0: expected identifier">;
def warn_cxx98_compat_decltype : Warning<
Expand Down
10 changes: 5 additions & 5 deletions clang/include/clang/Basic/DiagnosticSemaKinds.td
Original file line number Diff line number Diff line change
Expand Up @@ -12733,19 +12733,19 @@ def err_acc_size_expr_value
"OpenACC 'tile' clause size expression must be %select{an asterisk "
"or a constant expression|positive integer value, evaluated to %1}0">;
def err_acc_invalid_in_loop
: Error<"%select{OpenACC '%2' construct|while loop|do loop}0 cannot appear "
"in intervening code of a 'loop' with a '%1' clause">;
: Error<"%select{OpenACC '%3' construct|while loop|do loop}0 cannot appear "
"in intervening code of a '%1' with a '%2' clause">;
def note_acc_active_clause_here
: Note<"active '%0' clause defined here">;
def err_acc_clause_multiple_loops
: Error<"more than one for-loop in a loop associated with OpenACC 'loop' "
"construct with a '%select{collapse|tile}0' clause">;
: Error<"more than one for-loop in a loop associated with OpenACC '%0' "
"construct with a '%1' clause">;
def err_acc_insufficient_loops
: Error<"'%0' clause specifies a loop count greater than the number "
"of available loops">;
def err_acc_intervening_code
: Error<"inner loops must be tightly nested inside a '%0' clause on "
"a 'loop' construct">;
"a '%1' construct">;
def err_acc_gang_multiple_elt
: Error<"OpenACC 'gang' clause may have at most one %select{unnamed or "
"'num'|'dim'|'static'}0 argument">;
Expand Down
3 changes: 3 additions & 0 deletions clang/include/clang/Basic/Features.def
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ FEATURE(c_atomic, LangOpts.C11)
FEATURE(c_generic_selections, LangOpts.C11)
FEATURE(c_static_assert, LangOpts.C11)
FEATURE(c_thread_local, LangOpts.C11 &&PP.getTargetInfo().isTLSSupported())
// C23 features
FEATURE(c_fixed_enum, LangOpts.C23)
// C++11 features
FEATURE(cxx_access_control_sfinae, LangOpts.CPlusPlus11)
FEATURE(cxx_alias_templates, LangOpts.CPlusPlus11)
Expand Down Expand Up @@ -269,6 +271,7 @@ EXTENSION(c_static_assert, true)
EXTENSION(c_thread_local, PP.getTargetInfo().isTLSSupported())
// C23 features supported by other languages as extensions
EXTENSION(c_attributes, true)
EXTENSION(c_fixed_enum, true)
// C++11 features supported by other languages as extensions.
EXTENSION(cxx_atomic, LangOpts.CPlusPlus)
EXTENSION(cxx_default_function_template_args, LangOpts.CPlusPlus)
Expand Down
Loading