[clangd] Inactive regions support via dedicated protocol

This implements the server side of the approach discussed at
clangd/vscode-clangd#193 (comment)

Differential Revision: https://reviews.llvm.org/D143974

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -494,6 +494,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
     BackgroundIndexProgressState = BackgroundIndexProgress::Empty;
   BackgroundIndexSkipCreate = Params.capabilities.ImplicitProgressCreation;
   Opts.ImplicitCancellation = !Params.capabilities.CancelsStaleRequests;
+  Opts.PublishInactiveRegions = Params.capabilities.InactiveRegions;
 
   if (Opts.UseDirBasedCDB) {
     DirectoryBasedGlobalCompilationDatabase::Options CDBOpts(TFS);
@@ -582,6 +583,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
       {"memoryUsageProvider", true},           // clangd extension
       {"compilationDatabase",                  // clangd extension
        llvm::json::Object{{"automaticReload", true}}},
+      {"inactiveRegionsProvider", true}, // clangd extension
       {"callHierarchyProvider", true},
       {"clangdInlayHintsProvider", true},
       {"inlayHintProvider", true},
@@ -1625,6 +1627,8 @@ void ClangdLSPServer::bindMethods(LSPBinder &Bind,
 
   ApplyWorkspaceEdit = Bind.outgoingMethod("workspace/applyEdit");
   PublishDiagnostics = Bind.outgoingNotification("textDocument/publishDiagnostics");
+  if (Caps.InactiveRegions)
+    PublishInactiveRegions = Bind.outgoingNotification("textDocument/inactiveRegions");
   ShowMessage = Bind.outgoingNotification("window/showMessage");
   NotifyFileStatus = Bind.outgoingNotification("textDocument/clangd.fileStatus");
   CreateWorkDoneProgress = Bind.outgoingMethod("window/workDoneProgress/create");
@@ -1722,6 +1726,15 @@ void ClangdLSPServer::onDiagnosticsReady(PathRef File, llvm::StringRef Version,
   PublishDiagnostics(Notification);
 }
 
+void ClangdLSPServer::onInactiveRegionsReady(
+    PathRef File, std::vector<Range> InactiveRegions) {
+  InactiveRegionsParams Notification;
+  Notification.TextDocument = {URIForFile::canonicalize(File, /*TUPath=*/File)};
+  Notification.InactiveRegions = std::move(InactiveRegions);
+
+  PublishInactiveRegions(Notification);
+}
+
 void ClangdLSPServer::onBackgroundIndexProgress(
     const BackgroundQueue::Stats &Stats) {
   static const char ProgressToken[] = "backgroundIndexProgress";

clang-tools-extra/clangd/ClangdLSPServer.h

@@ -83,6 +83,8 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
   void onFileUpdated(PathRef File, const TUStatus &Status) override;
   void onBackgroundIndexProgress(const BackgroundQueue::Stats &Stats) override;
   void onSemanticsMaybeChanged(PathRef File) override;
+  void onInactiveRegionsReady(PathRef File,
+                              std::vector<Range> InactiveRegions) override;
 
   // LSP methods. Notifications have signature void(const Params&).
   // Calls have signature void(const Params&, Callback<Response>).
@@ -180,6 +182,7 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
   LSPBinder::OutgoingNotification<ShowMessageParams> ShowMessage;
   LSPBinder::OutgoingNotification<PublishDiagnosticsParams> PublishDiagnostics;
   LSPBinder::OutgoingNotification<FileStatus> NotifyFileStatus;
+  LSPBinder::OutgoingNotification<InactiveRegionsParams> PublishInactiveRegions;
   LSPBinder::OutgoingMethod<WorkDoneProgressCreateParams, std::nullptr_t>
       CreateWorkDoneProgress;
   LSPBinder::OutgoingNotification<ProgressParams<WorkDoneProgressBegin>>

clang-tools-extra/clangd/ClangdServer.cpp

@@ -61,9 +61,11 @@ namespace {
 struct UpdateIndexCallbacks : public ParsingCallbacks {
   UpdateIndexCallbacks(FileIndex *FIndex,
                        ClangdServer::Callbacks *ServerCallbacks,
-                       const ThreadsafeFS &TFS, AsyncTaskRunner *Tasks)
+                       const ThreadsafeFS &TFS, AsyncTaskRunner *Tasks,
+                       bool CollectInactiveRegions)
       : FIndex(FIndex), ServerCallbacks(ServerCallbacks), TFS(TFS),
-        Stdlib{std::make_shared<StdLibSet>()}, Tasks(Tasks) {}
+        Stdlib{std::make_shared<StdLibSet>()}, Tasks(Tasks),
+        CollectInactiveRegions(CollectInactiveRegions) {}
 
   void onPreambleAST(PathRef Path, llvm::StringRef Version,
                      const CompilerInvocation &CI, ASTContext &Ctx,
@@ -113,6 +115,10 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
       Publish([&]() {
         ServerCallbacks->onDiagnosticsReady(Path, AST.version(),
                                             std::move(Diagnostics));
+        if (CollectInactiveRegions) {
+          ServerCallbacks->onInactiveRegionsReady(
+              Path, std::move(AST.getMacros().SkippedRanges));
+        }
       });
   }
 
@@ -139,6 +145,7 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
   const ThreadsafeFS &TFS;
   std::shared_ptr<StdLibSet> Stdlib;
   AsyncTaskRunner *Tasks;
+  bool CollectInactiveRegions;
 };
 
 class DraftStoreFS : public ThreadsafeFS {
@@ -189,6 +196,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
       LineFoldingOnly(Opts.LineFoldingOnly),
       PreambleParseForwardingFunctions(Opts.PreambleParseForwardingFunctions),
       ImportInsertions(Opts.ImportInsertions),
+      PublishInactiveRegions(Opts.PublishInactiveRegions),
       WorkspaceRoot(Opts.WorkspaceRoot),
       Transient(Opts.ImplicitCancellation ? TUScheduler::InvalidateOnUpdate
                                           : TUScheduler::NoInvalidation),
@@ -201,7 +209,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
   WorkScheduler.emplace(CDB, TUScheduler::Options(Opts),
                         std::make_unique<UpdateIndexCallbacks>(
                             DynamicIdx.get(), Callbacks, TFS,
-                            IndexTasks ? &*IndexTasks : nullptr));
+                            IndexTasks ? &*IndexTasks : nullptr,
+                            PublishInactiveRegions));
   // Adds an index to the stack, at higher priority than existing indexes.
   auto AddIndex = [&](SymbolIndex *Idx) {
     if (this->Index != nullptr) {
@@ -956,12 +965,17 @@ void ClangdServer::documentLinks(PathRef File,
 
 void ClangdServer::semanticHighlights(
     PathRef File, Callback<std::vector<HighlightingToken>> CB) {
-  auto Action =
-      [CB = std::move(CB)](llvm::Expected<InputsAndAST> InpAST) mutable {
-        if (!InpAST)
-          return CB(InpAST.takeError());
-        CB(clangd::getSemanticHighlightings(InpAST->AST));
-      };
+
+  auto Action = [CB = std::move(CB),
+                 PublishInactiveRegions = PublishInactiveRegions](
+                    llvm::Expected<InputsAndAST> InpAST) mutable {
+    if (!InpAST)
+      return CB(InpAST.takeError());
+    // Include inactive regions in semantic highlighting tokens only if the
+    // client doesn't support a dedicated protocol for being informed about
+    // them.
+    CB(clangd::getSemanticHighlightings(InpAST->AST, !PublishInactiveRegions));
+  };
   WorkScheduler->runWithAST("SemanticHighlights", File, std::move(Action),
                             Transient);
 }

clang-tools-extra/clangd/ClangdServer.h

@@ -84,6 +84,11 @@ class ClangdServer {
     /// build finishes, we can provide more accurate semantic tokens, so we
     /// should tell the client to refresh.
     virtual void onSemanticsMaybeChanged(PathRef File) {}
+
+    /// Called by ClangdServer when some \p InactiveRegions for \p File are
+    /// ready.
+    virtual void onInactiveRegionsReady(PathRef File,
+                                        std::vector<Range> InactiveRegions) {}
   };
   /// Creates a context provider that loads and installs config.
   /// Errors in loading config are reported as diagnostics via Callbacks.
@@ -175,6 +180,10 @@ class ClangdServer {
     /// instead of #include.
     bool ImportInsertions = false;
 
+    /// Whether to collect and publish information about inactive preprocessor
+    /// regions in the document.
+    bool PublishInactiveRegions = false;
+
     explicit operator TUScheduler::Options() const;
   };
   // Sensible default options for use in tests.
@@ -440,6 +449,8 @@ class ClangdServer {
 
   bool ImportInsertions = false;
 
+  bool PublishInactiveRegions = false;
+
   // GUARDED_BY(CachedCompletionFuzzyFindRequestMutex)
   llvm::StringMap<std::optional<FuzzyFindRequest>>
       CachedCompletionFuzzyFindRequestByFile;

clang-tools-extra/clangd/Protocol.cpp

@@ -338,6 +338,13 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R,
               SemanticHighlighting->getBoolean("semanticHighlighting"))
         R.TheiaSemanticHighlighting = *SemanticHighlightingSupport;
     }
+    if (auto *InactiveRegions =
+            TextDocument->getObject("inactiveRegionsCapabilities")) {
+      if (auto InactiveRegionsSupport =
+              InactiveRegions->getBoolean("inactiveRegions")) {
+        R.InactiveRegions = *InactiveRegionsSupport;
+      }
+    }
     if (TextDocument->getObject("semanticTokens"))
       R.SemanticTokens = true;
     if (auto *Diagnostics = TextDocument->getObject("publishDiagnostics")) {
@@ -1174,6 +1181,12 @@ bool fromJSON(const llvm::json::Value &Params, SemanticTokensDeltaParams &R,
          O.map("previousResultId", R.previousResultId);
 }
 
+llvm::json::Value toJSON(const InactiveRegionsParams &InactiveRegions) {
+  return llvm::json::Object{
+      {"textDocument", InactiveRegions.TextDocument},
+      {"regions", std::move(InactiveRegions.InactiveRegions)}};
+}
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &O,
                               const DocumentHighlight &V) {
   O << V.range;

clang-tools-extra/clangd/Protocol.h

@@ -516,6 +516,10 @@ struct ClientCapabilities {
   /// Whether the client implementation supports a refresh request sent from the
   /// server to the client.
   bool SemanticTokenRefreshSupport = false;
+
+  /// Whether the client supports the textDocument/inactiveRegions
+  /// notification. This is a clangd extension.
+  bool InactiveRegions = false;
 };
 bool fromJSON(const llvm::json::Value &, ClientCapabilities &,
               llvm::json::Path);
@@ -1736,6 +1740,16 @@ struct SemanticTokensOrDelta {
 };
 llvm::json::Value toJSON(const SemanticTokensOrDelta &);
 
+/// Parameters for the inactive regions (server-side) push notification.
+/// This is a clangd extension.
+struct InactiveRegionsParams {
+  /// The textdocument these inactive regions belong to.
+  TextDocumentIdentifier TextDocument;
+  /// The inactive regions that should be sent.
+  std::vector<Range> InactiveRegions;
+};
+llvm::json::Value toJSON(const InactiveRegionsParams &InactiveRegions);
+
 struct SelectionRangeParams {
   /// The text document.
   TextDocumentIdentifier textDocument;

clang-tools-extra/clangd/SemanticHighlighting.cpp

@@ -357,9 +357,10 @@ resolveConflict(ArrayRef<HighlightingToken> Tokens) {
 /// Consumes source locations and maps them to text ranges for highlightings.
 class HighlightingsBuilder {
 public:
-  HighlightingsBuilder(const ParsedAST &AST)
+  HighlightingsBuilder(const ParsedAST &AST, bool IncludeInactiveRegionTokens)
       : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
-        LangOpts(AST.getLangOpts()) {}
+        LangOpts(AST.getLangOpts()),
+        IncludeInactiveRegionTokens(IncludeInactiveRegionTokens) {}
 
   HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
     auto Range = getRangeForSourceLocation(Loc);
@@ -458,6 +459,9 @@ class HighlightingsBuilder {
       TokRef = TokRef.drop_front(Conflicting.size());
     }
 
+    if (!IncludeInactiveRegionTokens)
+      return NonConflicting;
+
     const auto &SM = AST.getSourceManager();
     StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
 
@@ -531,6 +535,7 @@ class HighlightingsBuilder {
   const syntax::TokenBuffer &TB;
   const SourceManager &SourceMgr;
   const LangOptions &LangOpts;
+  bool IncludeInactiveRegionTokens;
   std::vector<HighlightingToken> Tokens;
   std::map<Range, llvm::SmallVector<HighlightingModifier, 1>> ExtraModifiers;
   const HeuristicResolver *Resolver = nullptr;
@@ -1096,10 +1101,11 @@ class CollectExtraHighlightings
 };
 } // namespace
 
-std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
+std::vector<HighlightingToken>
+getSemanticHighlightings(ParsedAST &AST, bool IncludeInactiveRegionTokens) {
   auto &C = AST.getASTContext();
   // Add highlightings for AST nodes.
-  HighlightingsBuilder Builder(AST);
+  HighlightingsBuilder Builder(AST, IncludeInactiveRegionTokens);
   // Highlight 'decltype' and 'auto' as their underlying types.
   CollectExtraHighlightings(Builder).TraverseAST(C);
   // Highlight all decls and references coming from the AST.

clang-tools-extra/clangd/SemanticHighlighting.h

@@ -106,7 +106,8 @@ bool operator<(const HighlightingToken &L, const HighlightingToken &R);
 
 // Returns all HighlightingTokens from an AST. Only generates highlights for the
 // main AST.
-std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
+std::vector<HighlightingToken>
+getSemanticHighlightings(ParsedAST &AST, bool IncludeInactiveRegionTokens);
 
 std::vector<SemanticToken> toSemanticTokens(llvm::ArrayRef<HighlightingToken>,
                                             llvm::StringRef Code);

clang-tools-extra/clangd/refactor/tweaks/AnnotateHighlightings.cpp

@@ -47,14 +47,16 @@ Expected<Tweak::Effect> AnnotateHighlightings::apply(const Selection &Inputs) {
     // Now we hit the TUDecl case where commonAncestor() returns null
     // intendedly. We only annotate tokens in the main file, so use the default
     // traversal scope (which is the top level decls of the main file).
-    HighlightingTokens = getSemanticHighlightings(*Inputs.AST);
+    HighlightingTokens = getSemanticHighlightings(
+        *Inputs.AST, /*IncludeInactiveRegionTokens=*/true);
   } else {
     // Store the existing scopes.
     const auto &BackupScopes = Inputs.AST->getASTContext().getTraversalScope();
     // Narrow the traversal scope to the selected node.
     Inputs.AST->getASTContext().setTraversalScope(
         {const_cast<Decl *>(CommonDecl)});
-    HighlightingTokens = getSemanticHighlightings(*Inputs.AST);
+    HighlightingTokens = getSemanticHighlightings(
+        *Inputs.AST, /*IncludeInactiveRegionTokens=*/true);
     // Restore the traversal scope.
     Inputs.AST->getASTContext().setTraversalScope(BackupScopes);
   }

clang-tools-extra/clangd/tool/Check.cpp

@@ -344,7 +344,8 @@ class Checker {
 
   void buildSemanticHighlighting(std::optional<Range> LineRange) {
     log("Building semantic highlighting");
-    auto Highlights = getSemanticHighlightings(*AST);
+    auto Highlights =
+        getSemanticHighlightings(*AST, /*IncludeInactiveRegionTokens=*/true);
     for (const auto HL : Highlights)
       if (!LineRange || LineRange->contains(HL.R))
         vlog(" {0} {1} {2}", HL.R, HL.Kind, HL.Modifiers);

clang-tools-extra/clangd/unittests/ClangdTests.cpp

@@ -1310,6 +1310,50 @@ TEST(ClangdServer, RespectsTweakFormatting) {
   });
   N.wait();
 }
+
+TEST(ClangdServer, InactiveRegions) {
+  struct InactiveRegionsCallback : ClangdServer::Callbacks {
+    std::vector<std::vector<Range>> FoundInactiveRegions;
+
+    void onInactiveRegionsReady(PathRef FIle,
+                                std::vector<Range> InactiveRegions) override {
+      FoundInactiveRegions.push_back(std::move(InactiveRegions));
+    }
+  };
+
+  MockFS FS;
+  MockCompilationDatabase CDB;
+  CDB.ExtraClangFlags.push_back("-DCMDMACRO");
+  auto Opts = ClangdServer::optsForTest();
+  Opts.PublishInactiveRegions = true;
+  InactiveRegionsCallback Callback;
+  ClangdServer Server(CDB, FS, Opts, &Callback);
+  Annotations Source(R"cpp(
+#define PREAMBLEMACRO 42
+#if PREAMBLEMACRO > 40
+  #define ACTIVE
+$inactive1[[#else
+  #define INACTIVE
+#endif]]
+int endPreamble;
+$inactive2[[#ifndef CMDMACRO
+    int inactiveInt;
+#endif]]
+#undef CMDMACRO
+$inactive3[[#ifdef CMDMACRO
+  int inactiveInt2;
+#else]]
+  int activeInt;
+#endif
+  )cpp");
+  Server.addDocument(testPath("foo.cpp"), Source.code());
+  ASSERT_TRUE(Server.blockUntilIdleForTest());
+  EXPECT_THAT(Callback.FoundInactiveRegions,
+              ElementsAre(ElementsAre(Source.range("inactive1"),
+                                      Source.range("inactive2"),
+                                      Source.range("inactive3"))));
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

@@ -89,7 +89,8 @@ void checkHighlightings(llvm::StringRef Code,
   for (auto File : AdditionalFiles)
     TU.AdditionalFiles.insert({File.first, std::string(File.second)});
   auto AST = TU.build();
-  auto Actual = getSemanticHighlightings(AST);
+  auto Actual =
+      getSemanticHighlightings(AST, /*IncludeInactiveRegionTokens=*/true);
   for (auto &Token : Actual)
     Token.Modifiers &= ModifierMask;