165 changes: 133 additions & 32 deletions clang-tools-extra/clangd/Preamble.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
#include "Preamble.h"
#include "Compiler.h"
#include "Config.h"
#include "Diagnostics.h"
#include "Headers.h"
#include "Protocol.h"
#include "SourceCode.h"
#include "clang-include-cleaner/Record.h"
#include "support/Logger.h"
Expand All @@ -35,6 +37,9 @@
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
Expand All @@ -43,8 +48,9 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include <iterator>
#include <cstddef>
#include <memory>
#include <optional>
#include <string>
#include <system_error>
#include <utility>
Expand All @@ -53,7 +59,6 @@
namespace clang {
namespace clangd {
namespace {
constexpr llvm::StringLiteral PreamblePatchHeaderName = "__preamble_patch__.h";

bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
const tooling::CompileCommand &RHS) {
Expand Down Expand Up @@ -213,6 +218,9 @@ struct TextualPPDirective {
// Full text that's representing the directive, including the `#`.
std::string Text;
unsigned Offset;
tok::PPKeywordKind Directive = tok::PPKeywordKind::pp_not_keyword;
// Name of the macro being defined in the case of a #define directive.
std::string MacroName;

bool operator==(const TextualPPDirective &RHS) const {
return std::tie(DirectiveLine, Offset, Text) ==
Expand Down Expand Up @@ -283,6 +291,8 @@ struct DirectiveCollector : public PPCallbacks {
return;
TextualDirectives.emplace_back();
TextualPPDirective &TD = TextualDirectives.back();
TD.Directive = tok::pp_define;
TD.MacroName = MacroNameTok.getIdentifierInfo()->getName().str();

const auto *MI = MD->getMacroInfo();
TD.Text =
Expand All @@ -302,6 +312,8 @@ struct DirectiveCollector : public PPCallbacks {
struct ScannedPreamble {
std::vector<Inclusion> Includes;
std::vector<TextualPPDirective> TextualDirectives;
// Literal lines of the preamble contents.
std::vector<llvm::StringRef> Lines;
PreambleBounds Bounds = {0, false};
};

Expand All @@ -328,7 +340,7 @@ scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
if (!CI)
return error("failed to create compiler invocation");
CI->getDiagnosticOpts().IgnoreWarnings = true;
auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Contents);
auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(PI.Contents);
// This means we're scanning (though not preprocessing) the preamble section
// twice. However, it's important to precisely follow the preamble bounds used
// elsewhere.
Expand Down Expand Up @@ -359,6 +371,7 @@ scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
return std::move(Err);
Action.EndSourceFile();
SP.Includes = std::move(Includes.MainFileIncludes);
llvm::append_range(SP.Lines, llvm::split(Contents, "\n"));
return SP;
}

Expand All @@ -376,12 +389,6 @@ const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
llvm_unreachable("not an include directive");
}

// Checks whether \p FileName is a valid spelling of main file.
bool isMainFile(llvm::StringRef FileName, const SourceManager &SM) {
auto FE = SM.getFileManager().getFile(FileName);
return FE && *FE == SM.getFileEntryForID(SM.getMainFileID());
}

// Accumulating wall time timer. Similar to llvm::Timer, but much cheaper,
// it only tracks wall time.
// Since this is a generic timer, We may want to move this to support/ if we
Expand Down Expand Up @@ -467,6 +474,93 @@ class TimerFS : public llvm::vfs::ProxyFileSystem {
WallTimer Timer;
};

// Helpers for patching diagnostics between two versions of file contents.
class DiagPatcher {
llvm::ArrayRef<llvm::StringRef> OldLines;
llvm::ArrayRef<llvm::StringRef> CurrentLines;
llvm::StringMap<llvm::SmallVector<int>> CurrentContentsToLine;

// Translates a range from old lines to current lines.
// Finds the consecutive set of lines that corresponds to the same contents in
// old and current, and applies the same translation to the range.
// Returns true if translation succeeded.
bool translateRange(Range &R) {
int OldStart = R.start.line;
int OldEnd = R.end.line;
assert(OldStart <= OldEnd);

size_t RangeLen = OldEnd - OldStart + 1;
auto RangeContents = OldLines.slice(OldStart).take_front(RangeLen);
// Make sure the whole range is covered in old contents.
if (RangeContents.size() < RangeLen)
return false;

std::optional<int> Closest;
for (int AlternateLine : CurrentContentsToLine.lookup(RangeContents[0])) {
// Check if AlternateLine matches all lines in the range.
if (RangeContents !=
CurrentLines.slice(AlternateLine).take_front(RangeLen))
continue;
int Delta = AlternateLine - OldStart;
if (!Closest.has_value() || abs(Delta) < abs(*Closest))
Closest = Delta;
}
// Couldn't find any viable matches in the current contents.
if (!Closest.has_value())
return false;
R.start.line += *Closest;
R.end.line += *Closest;
return true;
}

// Translates a Note by patching its range when inside main file. Returns true
// on success.
bool translateNote(Note &N) {
if (!N.InsideMainFile)
return true;
if (translateRange(N.Range))
return true;
return false;
}

// Tries to translate all the edit ranges inside the fix. Returns true on
// success. On failure fixes might be in an invalid state.
bool translateFix(Fix &F) {
return llvm::all_of(
F.Edits, [this](TextEdit &E) { return translateRange(E.range); });
}

public:
DiagPatcher(llvm::ArrayRef<llvm::StringRef> OldLines,
llvm::ArrayRef<llvm::StringRef> CurrentLines) {
this->OldLines = OldLines;
this->CurrentLines = CurrentLines;
for (int Line = 0, E = CurrentLines.size(); Line != E; ++Line) {
llvm::StringRef Contents = CurrentLines[Line];
CurrentContentsToLine[Contents].push_back(Line);
}
}
// Translate diagnostic by moving its main range to new location (if inside
// the main file). Preserve all the notes and fixes that can be translated to
// new contents.
// Drops the whole diagnostic if main range can't be patched.
std::optional<Diag> translateDiag(const Diag &D) {
Range NewRange = D.Range;
// Patch range if it's inside main file.
if (D.InsideMainFile && !translateRange(NewRange)) {
// Drop the diagnostic if we couldn't patch the range.
return std::nullopt;
}

Diag NewD = D;
NewD.Range = NewRange;
// Translate ranges inside notes and fixes too, dropping the ones that are
// no longer relevant.
llvm::erase_if(NewD.Notes, [this](Note &N) { return !translateNote(N); });
llvm::erase_if(NewD.Fixes, [this](Fix &F) { return !translateFix(F); });
return NewD;
}
};
} // namespace

std::shared_ptr<const PreambleData>
Expand Down Expand Up @@ -560,8 +654,8 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,

if (BuiltPreamble) {
log("Built preamble of size {0} for file {1} version {2} in {3} seconds",
BuiltPreamble->getSize(), FileName, Inputs.Version,
PreambleTimer.getTime());
BuiltPreamble->getSize(), FileName, Inputs.Version,
PreambleTimer.getTime());
std::vector<Diag> Diags = PreambleDiagnostics.take();
auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble));
Result->Version = Inputs.Version;
Expand Down Expand Up @@ -614,6 +708,22 @@ void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
}
}

// Translate diagnostics from baseline into modified for the lines that have the
// same spelling.
static std::vector<Diag> patchDiags(llvm::ArrayRef<Diag> BaselineDiags,
const ScannedPreamble &BaselineScan,
const ScannedPreamble &ModifiedScan) {
std::vector<Diag> PatchedDiags;
if (BaselineDiags.empty())
return PatchedDiags;
DiagPatcher Patcher(BaselineScan.Lines, ModifiedScan.Lines);
for (auto &D : BaselineDiags) {
if (auto NewD = Patcher.translateDiag(D))
PatchedDiags.emplace_back(std::move(*NewD));
}
return PatchedDiags;
}

PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
const ParseInputs &Modified,
const PreambleData &Baseline,
Expand All @@ -631,7 +741,7 @@ PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
// there's nothing to do but generate an empty patch.
auto BaselineScan = scanPreamble(
// Contents needs to be null-terminated.
Baseline.Preamble.getContents().str(), Modified.CompileCommand);
Baseline.Preamble.getContents(), Modified.CompileCommand);
if (!BaselineScan) {
elog("Failed to scan baseline of {0}: {1}", FileName,
BaselineScan.takeError());
Expand All @@ -655,7 +765,7 @@ PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
// This shouldn't coincide with any real file name.
llvm::SmallString<128> PatchName;
llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
PreamblePatchHeaderName);
PreamblePatch::HeaderName);
PP.PatchFileName = PatchName.str().str();
PP.ModifiedBounds = ModifiedScan->Bounds;

Expand Down Expand Up @@ -724,10 +834,16 @@ PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
// reduce complexity. The former might cause problems because scanning is
// imprecise and might pick directives from disabled regions.
for (const auto &TD : ModifiedScan->TextualDirectives) {
// Introduce an #undef directive before #defines to suppress any
// re-definition warnings.
if (TD.Directive == tok::pp_define)
Patch << "#undef " << TD.MacroName << '\n';
Patch << "#line " << TD.DirectiveLine << '\n';
Patch << TD.Text << '\n';
}
}

PP.PatchedDiags = patchDiags(Baseline.Diags, *BaselineScan, *ModifiedScan);
dlog("Created preamble patch: {0}", Patch.str());
Patch.flush();
return PP;
Expand Down Expand Up @@ -769,28 +885,13 @@ PreamblePatch PreamblePatch::unmodified(const PreambleData &Preamble) {
PreamblePatch PP;
PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
PP.ModifiedBounds = Preamble.Preamble.getBounds();
PP.PatchedDiags = Preamble.Diags;
return PP;
}

SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
const SourceManager &SM) {
auto DefFile = SM.getFileID(Loc);
if (auto FE = SM.getFileEntryRefForID(DefFile)) {
auto IncludeLoc = SM.getIncludeLoc(DefFile);
// Preamble patch is included inside the builtin file.
if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
FE->getName().endswith(PreamblePatchHeaderName)) {
auto Presumed = SM.getPresumedLoc(Loc);
// Check that line directive is pointing at main file.
if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
isMainFile(Presumed.getFilename(), SM)) {
Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(),
Presumed.getColumn());
}
}
}
return Loc;
bool PreamblePatch::preserveDiagnostics() const {
return PatchContents.empty() ||
Config::current().Diagnostics.AllowStalePreamble;
}

} // namespace clangd
} // namespace clang
19 changes: 11 additions & 8 deletions clang-tools-extra/clangd/Preamble.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "clang/Frontend/PrecompiledPreamble.h"
#include "clang/Lex/Lexer.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringRef.h"

#include <memory>
Expand Down Expand Up @@ -155,7 +156,12 @@ class PreamblePatch {
llvm::StringRef text() const { return PatchContents; }

/// Whether diagnostics generated using this patch are trustable.
bool preserveDiagnostics() const { return PatchContents.empty(); }
bool preserveDiagnostics() const;

/// Returns diag locations for Modified contents.
llvm::ArrayRef<Diag> patchedDiags() const { return PatchedDiags; }

static constexpr llvm::StringLiteral HeaderName = "__preamble_patch__.h";

private:
static PreamblePatch create(llvm::StringRef FileName,
Expand All @@ -166,17 +172,14 @@ class PreamblePatch {
PreamblePatch() = default;
std::string PatchContents;
std::string PatchFileName;
/// Includes that are present in both \p Baseline and \p Modified. Used for
/// patching includes of baseline preamble.
// Includes that are present in both Baseline and Modified. Used for
// patching includes of baseline preamble.
std::vector<Inclusion> PreambleIncludes;
// Diags that were attached to a line preserved in Modified contents.
std::vector<Diag> PatchedDiags;
PreambleBounds ModifiedBounds = {0, false};
};

/// Translates locations inside preamble patch to their main-file equivalent
/// using presumed locations. Returns \p Loc if it isn't inside preamble patch.
SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
const SourceManager &SM);

} // namespace clangd
} // namespace clang

Expand Down
25 changes: 25 additions & 0 deletions clang-tools-extra/clangd/SourceCode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,12 @@ llvm::SmallVector<llvm::StringRef> ancestorNamespaces(llvm::StringRef NS) {
return Results;
}

// Checks whether \p FileName is a valid spelling of main file.
bool isMainFile(llvm::StringRef FileName, const SourceManager &SM) {
auto FE = SM.getFileManager().getFile(FileName);
return FE && *FE == SM.getFileEntryForID(SM.getMainFileID());
}

} // namespace

std::vector<std::string> visibleNamespaces(llvm::StringRef Code,
Expand Down Expand Up @@ -1219,5 +1225,24 @@ bool isProtoFile(SourceLocation Loc, const SourceManager &SM) {
return SM.getBufferData(FID).startswith(ProtoHeaderComment);
}

SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
const SourceManager &SM) {
auto DefFile = SM.getFileID(Loc);
if (auto FE = SM.getFileEntryRefForID(DefFile)) {
auto IncludeLoc = SM.getIncludeLoc(DefFile);
// Preamble patch is included inside the builtin file.
if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
FE->getName().endswith(PreamblePatch::HeaderName)) {
auto Presumed = SM.getPresumedLoc(Loc);
// Check that line directive is pointing at main file.
if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
isMainFile(Presumed.getFilename(), SM)) {
Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(),
Presumed.getColumn());
}
}
}
return Loc;
}
} // namespace clangd
} // namespace clang
4 changes: 4 additions & 0 deletions clang-tools-extra/clangd/SourceCode.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,10 @@ inline bool isReservedName(llvm::StringRef Name) {
(isUppercase(Name[1]) || Name[1] == '_');
}

/// Translates locations inside preamble patch to their main-file equivalent
/// using presumed locations. Returns \p Loc if it isn't inside preamble patch.
SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
const SourceManager &SM);
} // namespace clangd
} // namespace clang
#endif
28 changes: 26 additions & 2 deletions clang-tools-extra/clangd/TUScheduler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
#include "TUScheduler.h"
#include "CompileCommands.h"
#include "Compiler.h"
#include "Config.h"
#include "Diagnostics.h"
#include "GlobalCompilationDatabase.h"
#include "ParsedAST.h"
Expand Down Expand Up @@ -938,8 +939,19 @@ void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags,
return;
}

PreamblePeer.update(std::move(Invocation), std::move(Inputs),
std::move(CompilerInvocationDiags), WantDiags);
// Inform preamble peer, before attempting to build diagnostics so that they
// can be built concurrently.
PreamblePeer.update(std::make_unique<CompilerInvocation>(*Invocation),
Inputs, CompilerInvocationDiags, WantDiags);

// Emit diagnostics from (possibly) stale preamble while waiting for a
// rebuild. Newly built preamble cannot emit diagnostics before this call
// finishes (ast callbacks are called from astpeer thread), hence we
// gurantee eventual consistency.
if (LatestPreamble && Config::current().Diagnostics.AllowStalePreamble)
generateDiagnostics(std::move(Invocation), std::move(Inputs),
std::move(CompilerInvocationDiags));

std::unique_lock<std::mutex> Lock(Mutex);
PreambleCV.wait(Lock, [this] {
// Block until we reiceve a preamble request, unless a preamble already
Expand Down Expand Up @@ -1118,6 +1130,18 @@ void ASTWorker::updatePreamble(std::unique_ptr<CompilerInvocation> CI,
// We only need to build the AST if diagnostics were requested.
if (WantDiags == WantDiagnostics::No)
return;
// The file may have been edited since we started building this preamble.
// If diagnostics need a fresh preamble, we must use the old version that
// matches the preamble. We make forward progress as updatePreamble()
// receives increasing versions, and this is the only place we emit
// diagnostics.
// If diagnostics can use a stale preamble, we use the current contents of
// the file instead. This provides more up-to-date diagnostics, and avoids
// diagnostics going backwards (we may have already emitted staler-preamble
// diagnostics for the new version). We still have eventual consistency: at
// some point updatePreamble() will catch up to the current file.
if (Config::current().Diagnostics.AllowStalePreamble)
PI = FileInputs;
// Report diagnostics with the new preamble to ensure progress. Otherwise
// diagnostics might get stale indefinitely if user keeps invalidating the
// preamble.
Expand Down
15 changes: 15 additions & 0 deletions clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -543,6 +543,21 @@ TEST_F(ConfigCompileTests, Style) {
EXPECT_TRUE(compileAndApply());
EXPECT_THAT(Conf.Style.FullyQualifiedNamespaces, ElementsAre("foo", "bar"));
}

TEST_F(ConfigCompileTests, AllowDiagsFromStalePreamble) {
Frag = {};
EXPECT_TRUE(compileAndApply());
// Off by default.
EXPECT_EQ(Conf.Diagnostics.AllowStalePreamble, false);

Frag.Diagnostics.AllowStalePreamble.emplace(true);
EXPECT_TRUE(compileAndApply());
EXPECT_EQ(Conf.Diagnostics.AllowStalePreamble, true);

Frag.Diagnostics.AllowStalePreamble.emplace(false);
EXPECT_TRUE(compileAndApply());
EXPECT_EQ(Conf.Diagnostics.AllowStalePreamble, false);
}
} // namespace
} // namespace config
} // namespace clangd
Expand Down
27 changes: 27 additions & 0 deletions clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,33 @@ TEST(ParseYAML, Style) {
EXPECT_THAT(Results[0].Style.FullyQualifiedNamespaces,
ElementsAre(val("foo"), val("bar")));
}

TEST(ParseYAML, DiagnosticsMode) {
CapturedDiags Diags;
{
Annotations YAML(R"yaml(
Diagnostics:
AllowStalePreamble: Yes)yaml");
auto Results =
Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback());
ASSERT_THAT(Diags.Diagnostics, IsEmpty());
ASSERT_EQ(Results.size(), 1u);
EXPECT_THAT(Results[0].Diagnostics.AllowStalePreamble,
llvm::ValueIs(val(true)));
}

{
Annotations YAML(R"yaml(
Diagnostics:
AllowStalePreamble: No)yaml");
auto Results =
Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback());
ASSERT_THAT(Diags.Diagnostics, IsEmpty());
ASSERT_EQ(Results.size(), 1u);
EXPECT_THAT(Results[0].Diagnostics.AllowStalePreamble,
llvm::ValueIs(val(false)));
}
}
} // namespace
} // namespace config
} // namespace clangd
Expand Down
4 changes: 2 additions & 2 deletions clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ TEST(IncludeCleaner, IWYUPragmas) {
void foo() {}
)cpp");
Config Cfg;
Cfg.Diagnostics.UnusedIncludes = Config::Experiment;
Cfg.Diagnostics.UnusedIncludes = Config::UnusedIncludesPolicy::Experiment;
WithContextValue Ctx(Config::Key, std::move(Cfg));
ParsedAST AST = TU.build();

Expand Down Expand Up @@ -627,7 +627,7 @@ TEST(IncludeCleaner, NoDiagsForObjC) {
TU.ExtraArgs.emplace_back("-xobjective-c");

Config Cfg;
Cfg.Diagnostics.UnusedIncludes = Config::Strict;
Cfg.Diagnostics.UnusedIncludes = Config::UnusedIncludesPolicy::Strict;
WithContextValue Ctx(Config::Key, std::move(Cfg));
ParsedAST AST = TU.build();
EXPECT_THAT(AST.getDiagnostics(), llvm::ValueIs(IsEmpty()));
Expand Down
236 changes: 232 additions & 4 deletions clang-tools-extra/clangd/unittests/PreambleTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@

#include "Annotations.h"
#include "Compiler.h"
#include "Config.h"
#include "Diagnostics.h"
#include "Headers.h"
#include "Hover.h"
#include "ParsedAST.h"
#include "Preamble.h"
#include "Protocol.h"
#include "SourceCode.h"
#include "TestFS.h"
#include "TestTU.h"
Expand All @@ -19,10 +23,13 @@
#include "clang/Format/Format.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Frontend/PrecompiledPreamble.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/ScopedPrinter.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Testing/Support/SupportHelpers.h"
#include "gmock/gmock.h"
#include "gtest/gtest-matchers.h"
#include "gtest/gtest.h"
Expand All @@ -31,10 +38,19 @@
#include <string>
#include <vector>

using testing::AllOf;
using testing::Contains;
using testing::ElementsAre;
using testing::ElementsAreArray;
using testing::Eq;
using testing::Field;
using testing::HasSubstr;
using testing::IsEmpty;
using testing::Matcher;
using testing::MatchesRegex;
using testing::Not;
using testing::UnorderedElementsAre;
using testing::UnorderedElementsAreArray;

namespace clang {
namespace clangd {
Expand Down Expand Up @@ -204,17 +220,20 @@ TEST(PreamblePatchTest, PatchesPreambleIncludes) {
Field(&Inclusion::FileKind, SrcMgr::CharacteristicKind::C_User))));
}

std::optional<ParsedAST> createPatchedAST(llvm::StringRef Baseline,
llvm::StringRef Modified) {
auto BaselinePreamble = TestTU::withCode(Baseline).preamble();
std::optional<ParsedAST>
createPatchedAST(llvm::StringRef Baseline, llvm::StringRef Modified,
llvm::StringMap<std::string> AdditionalFiles = {}) {
auto TU = TestTU::withCode(Baseline);
TU.AdditionalFiles = std::move(AdditionalFiles);
auto BaselinePreamble = TU.preamble();
if (!BaselinePreamble) {
ADD_FAILURE() << "Failed to build baseline preamble";
return std::nullopt;
}

IgnoreDiagnostics Diags;
MockFS FS;
auto TU = TestTU::withCode(Modified);
TU.Code = Modified.str();
auto CI = buildCompilerInvocation(TU.inputs(FS), Diags);
if (!CI) {
ADD_FAILURE() << "Failed to build compiler invocation";
Expand Down Expand Up @@ -270,6 +289,7 @@ TEST(PreamblePatchTest, Define) {
#define BAR
[[BAR]])cpp",
R"cpp(#line 0 ".*main.cpp"
#undef BAR
#line 2
#define BAR
)cpp",
Expand All @@ -281,6 +301,7 @@ TEST(PreamblePatchTest, Define) {
[[BAR]])cpp",
R"cpp(#line 0 ".*main.cpp"
#undef BAR
#line 2
#define BAR
)cpp",
Expand All @@ -292,6 +313,7 @@ TEST(PreamblePatchTest, Define) {
BAR
[[BAR]])cpp",
R"cpp(#line 0 ".*main.cpp"
#undef BAR
#line 3
#define BAR
)cpp",
Expand Down Expand Up @@ -324,8 +346,10 @@ TEST(PreamblePatchTest, OrderingPreserved) {
)cpp");

llvm::StringLiteral ExpectedPatch(R"cpp(#line 0 ".*main.cpp"
#undef BAR
#line 2
#define BAR\(X, Y\) X Y
#undef BAR
#line 3
#define BAR\(X\) X
)cpp");
Expand Down Expand Up @@ -599,6 +623,210 @@ TEST(PreamblePatch, NoopWhenNotRequested) {
TU.inputs(FS), *BaselinePreamble);
EXPECT_TRUE(PP.text().empty());
}

::testing::Matcher<const Diag &>
withNote(::testing::Matcher<Note> NoteMatcher) {
return Field(&Diag::Notes, ElementsAre(NoteMatcher));
}
MATCHER_P(Diag, Range, "Diag at " + llvm::to_string(Range)) {
return arg.Range == Range;
}
MATCHER_P2(Diag, Range, Name,
"Diag at " + llvm::to_string(Range) + " = [" + Name + "]") {
return arg.Range == Range && arg.Name == Name;
}

TEST(PreamblePatch, DiagnosticsFromMainASTAreInRightPlace) {
Config Cfg;
Cfg.Diagnostics.AllowStalePreamble = true;
WithContextValue WithCfg(Config::Key, std::move(Cfg));

{
Annotations Code("#define FOO");
// Check with removals from preamble.
Annotations NewCode("[[x]];/* error-ok */");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(),
ElementsAre(Diag(NewCode.range(), "missing_type_specifier")));
}
{
// Check with additions to preamble.
Annotations Code("#define FOO");
Annotations NewCode(R"(
#define FOO
#define BAR
[[x]];/* error-ok */)");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(),
ElementsAre(Diag(NewCode.range(), "missing_type_specifier")));
}
}

TEST(PreamblePatch, DiagnosticsToPreamble) {
Config Cfg;
Cfg.Diagnostics.AllowStalePreamble = true;
Cfg.Diagnostics.UnusedIncludes = Config::UnusedIncludesPolicy::Strict;
WithContextValue WithCfg(Config::Key, std::move(Cfg));

llvm::StringMap<std::string> AdditionalFiles;
AdditionalFiles["foo.h"] = "#pragma once";
AdditionalFiles["bar.h"] = "#pragma once";
{
Annotations Code(R"(
// Test comment
[[#include "foo.h"]])");
// Check with removals from preamble.
Annotations NewCode(R"([[# include "foo.h"]])");
auto AST = createPatchedAST(Code.code(), NewCode.code(), AdditionalFiles);
EXPECT_THAT(*AST->getDiagnostics(),
ElementsAre(Diag(NewCode.range(), "unused-includes")));
}
{
// Check with additions to preamble.
Annotations Code(R"(
// Test comment
[[#include "foo.h"]])");
Annotations NewCode(R"(
$bar[[#include "bar.h"]]
// Test comment
$foo[[#include "foo.h"]])");
auto AST = createPatchedAST(Code.code(), NewCode.code(), AdditionalFiles);
EXPECT_THAT(
*AST->getDiagnostics(),
UnorderedElementsAre(Diag(NewCode.range("bar"), "unused-includes"),
Diag(NewCode.range("foo"), "unused-includes")));
}
{
Annotations Code("#define [[FOO]] 1\n");
// Check ranges for notes.
Annotations NewCode(R"(#define BARXYZ 1
#define $foo1[[FOO]] 1
void foo();
#define $foo2[[FOO]] 2)");
auto AST = createPatchedAST(Code.code(), NewCode.code(), AdditionalFiles);
EXPECT_THAT(
*AST->getDiagnostics(),
ElementsAre(AllOf(Diag(NewCode.range("foo2"), "-Wmacro-redefined"),
withNote(Diag(NewCode.range("foo1"))))));
}
}

TEST(PreamblePatch, TranslatesDiagnosticsInPreamble) {
Config Cfg;
Cfg.Diagnostics.AllowStalePreamble = true;
WithContextValue WithCfg(Config::Key, std::move(Cfg));

{
// Check with additions to preamble.
Annotations Code("#include [[<foo>]]");
Annotations NewCode(R"(
#define BAR
#include [[<foo>]])");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(),
ElementsAre(Diag(NewCode.range(), "pp_file_not_found")));
}
{
// Check with removals from preamble.
Annotations Code(R"(
#define BAR
#include [[<foo>]])");
Annotations NewCode("#include [[<foo>]]");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(),
ElementsAre(Diag(NewCode.range(), "pp_file_not_found")));
}
{
// Drop line with diags.
Annotations Code("#include [[<foo>]]");
Annotations NewCode("#define BAR\n#define BAZ\n");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(), IsEmpty());
}
{
// Picks closest line in case of multiple alternatives.
Annotations Code("#include [[<foo>]]");
Annotations NewCode(R"(
#define BAR
#include [[<foo>]]
#define BAR
#include <foo>)");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(),
ElementsAre(Diag(NewCode.range(), "pp_file_not_found")));
}
{
// Drop diag if line spelling has changed.
Annotations Code("#include [[<foo>]]");
Annotations NewCode(" # include <foo>");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(), IsEmpty());
}
{
// Multiple lines.
Annotations Code(R"(
#define BAR
#include [[<fo\
o>]])");
Annotations NewCode(R"(#include [[<fo\
o>]])");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(),
ElementsAre(Diag(NewCode.range(), "pp_file_not_found")));
}
{
// Multiple lines with change.
Annotations Code(R"(
#define BAR
#include <fox>
#include [[<fo\
o>]])");
Annotations NewCode(R"(#include <fo\
x>)");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(), IsEmpty());
}
{
// Preserves notes.
Annotations Code(R"(
#define $note[[BAR]] 1
#define $main[[BAR]] 2)");
Annotations NewCode(R"(
#define BAZ 0
#define $note[[BAR]] 1
#define BAZ 0
#define $main[[BAR]] 2)");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(
*AST->getDiagnostics(),
ElementsAre(AllOf(Diag(NewCode.range("main"), "-Wmacro-redefined"),
withNote(Diag(NewCode.range("note"))))));
}
{
// Preserves diag without note.
Annotations Code(R"(
#define $note[[BAR]] 1
#define $main[[BAR]] 2)");
Annotations NewCode(R"(
#define $main[[BAR]] 2)");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(
*AST->getDiagnostics(),
ElementsAre(AllOf(Diag(NewCode.range("main"), "-Wmacro-redefined"),
Field(&Diag::Notes, IsEmpty()))));
}
{
// Make sure orphaned notes are not promoted to diags.
Annotations Code(R"(
#define $note[[BAR]] 1
#define $main[[BAR]] 2)");
Annotations NewCode(R"(
#define BAZ 0
#define BAR 1)");
auto AST = createPatchedAST(Code.code(), NewCode.code());
EXPECT_THAT(*AST->getDiagnostics(), IsEmpty());
}
}
} // namespace
} // namespace clangd
} // namespace clang
121 changes: 119 additions & 2 deletions clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

#include "Annotations.h"
#include "ClangdServer.h"
#include "Compiler.h"
#include "Config.h"
#include "Diagnostics.h"
#include "GlobalCompilationDatabase.h"
#include "Matchers.h"
Expand All @@ -21,7 +23,6 @@
#include "support/Path.h"
#include "support/TestTracer.h"
#include "support/Threading.h"
#include "support/ThreadsafeFS.h"
#include "clang/Basic/DiagnosticDriver.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/FunctionExtras.h"
Expand All @@ -31,19 +32,23 @@
#include "llvm/ADT/StringRef.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <utility>
#include <vector>

namespace clang {
namespace clangd {
namespace {

using ::testing::_;
using ::testing::AllOf;
using ::testing::AnyOf;
using ::testing::Contains;
Expand Down Expand Up @@ -1194,6 +1199,118 @@ TEST_F(TUSchedulerTests, OnlyPublishWhenPreambleIsBuilt) {
EXPECT_EQ(PreamblePublishCount, 2);
}

TEST_F(TUSchedulerTests, PublishWithStalePreamble) {
// Callbacks that blocks the preamble thread after the first preamble is
// built and stores preamble/main-file versions for diagnostics released.
class BlockPreambleThread : public ParsingCallbacks {
public:
using DiagsCB = std::function<void(ParsedAST &)>;
BlockPreambleThread(Notification &UnblockPreamble, DiagsCB CB)
: UnblockPreamble(UnblockPreamble), CB(std::move(CB)) {}

void onPreambleAST(PathRef Path, llvm::StringRef Version,
const CompilerInvocation &, ASTContext &Ctx,
Preprocessor &, const CanonicalIncludes &) override {
if (BuildBefore)
ASSERT_TRUE(UnblockPreamble.wait(timeoutSeconds(5)))
<< "Expected notification";
BuildBefore = true;
}

void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {
CB(AST);
}

void onFailedAST(PathRef File, llvm::StringRef Version,
std::vector<Diag> Diags, PublishFn Publish) override {
ADD_FAILURE() << "Received failed ast for: " << File << " with version "
<< Version << '\n';
}

private:
bool BuildBefore = false;
Notification &UnblockPreamble;
std::function<void(ParsedAST &)> CB;
};

// Helpers for issuing blocking update requests on a TUScheduler, whose
// onMainAST callback would call onDiagnostics.
class DiagCollector {
public:
void onDiagnostics(ParsedAST &AST) {
std::scoped_lock<std::mutex> Lock(DiagMu);
DiagVersions.emplace_back(
std::make_pair(AST.preambleVersion()->str(), AST.version().str()));
DiagsReceived.notify_all();
}

std::pair<std::string, std::string>
waitForNewDiags(TUScheduler &S, PathRef File, ParseInputs PI) {
std::unique_lock<std::mutex> Lock(DiagMu);
// Perform the update under the lock to make sure it isn't handled until
// we're waiting for it.
S.update(File, std::move(PI), WantDiagnostics::Auto);
size_t OldSize = DiagVersions.size();
bool ReceivedDiags = DiagsReceived.wait_for(
Lock, std::chrono::seconds(5),
[this, OldSize] { return OldSize + 1 == DiagVersions.size(); });
if (!ReceivedDiags) {
ADD_FAILURE() << "Timed out waiting for diags";
return {"invalid", "version"};
}
return DiagVersions.back();
}

std::vector<std::pair<std::string, std::string>> diagVersions() {
std::scoped_lock<std::mutex> Lock(DiagMu);
return DiagVersions;
}

private:
std::condition_variable DiagsReceived;
std::mutex DiagMu;
std::vector<std::pair</*PreambleVersion*/ std::string,
/*MainFileVersion*/ std::string>>
DiagVersions;
};

Config Cfg;
Cfg.Diagnostics.AllowStalePreamble = true;
WithContextValue WithCfg(Config::Key, std::move(Cfg));

DiagCollector Collector;
Notification UnblockPreamble;
auto DiagCallbacks = std::make_unique<BlockPreambleThread>(
UnblockPreamble,
[&Collector](ParsedAST &AST) { Collector.onDiagnostics(AST); });
TUScheduler S(CDB, optsForTest(), std::move(DiagCallbacks));
Path File = testPath("foo.cpp");
auto BlockForDiags = [&](ParseInputs PI) {
return Collector.waitForNewDiags(S, File, std::move(PI));
};

// Build first preamble.
auto PI = getInputs(File, "");
PI.Version = PI.Contents = "1";
ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "1"));

// Now preamble thread is blocked, so rest of the requests sees only the
// stale preamble.
PI.Version = "2";
PI.Contents = "#define BAR\n" + PI.Version;
ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "2"));

PI.Version = "3";
PI.Contents = "#define FOO\n" + PI.Version;
ASSERT_THAT(BlockForDiags(PI), testing::Pair("1", "3"));

UnblockPreamble.notify();
S.blockUntilIdle(timeoutSeconds(5));

// Make sure that we have eventual consistency.
EXPECT_THAT(Collector.diagVersions().back(), Pair(PI.Version, PI.Version));
}

// If a header file is missing from the CDB (or inferred using heuristics), and
// it's included by another open file, then we parse it using that files flags.
TEST_F(TUSchedulerTests, IncluderCache) {
Expand Down