update

Update

update

update

update

update

Update

update

fmt

Add MockDirectoryCompilationDatabase

add test

clang-tools-extra/clangd/CMakeLists.txt

@@ -99,12 +99,12 @@ add_clang_library(clangDaemon
   JSONTransport.cpp
   ModulesBuilder.cpp
   PathMapping.cpp
-  ProjectModules.cpp
   Protocol.cpp
   Quality.cpp
   ParsedAST.cpp
   Preamble.cpp
   RIFF.cpp
+  ScanningProjectModules.cpp
   Selection.cpp
   SemanticHighlighting.cpp
   SemanticSelection.cpp

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -52,6 +52,9 @@
 
 namespace clang {
 namespace clangd {
+
+extern llvm::cl::opt<bool> ExperimentalModulesSupport;
+
 namespace {
 // Tracks end-to-end latency of high level lsp calls. Measurements are in
 // seconds.
@@ -565,8 +568,10 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
   CDB.emplace(BaseCDB.get(), Params.initializationOptions.fallbackFlags,
               std::move(Mangler));
 
-  if (Opts.ExperimentalModulesSupport)
+  if (Opts.EnableExperimentalModulesSupport) {
     ModulesManager.emplace(*CDB);
+    Opts.ModulesManager = &*ModulesManager;
+  }
 
   {
     // Switch caller's context with LSPServer's background context. Since we
@@ -577,8 +582,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
     if (Opts.Encoding)
       WithOffsetEncoding.emplace(kCurrentOffsetEncoding, *Opts.Encoding);
     Server.emplace(*CDB, TFS, Opts,
-                   static_cast<ClangdServer::Callbacks *>(this),
-                   ModulesManager ? &*ModulesManager : nullptr);
+                   static_cast<ClangdServer::Callbacks *>(this));
   }
 
   llvm::json::Object ServerCaps{

clang-tools-extra/clangd/ClangdLSPServer.h

@@ -63,6 +63,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
 
     /// Limit the number of references returned (0 means no limit).
     size_t ReferencesLimit = 0;
+
+    /// Flag to hint the experimental modules support is enabled.
+    bool EnableExperimentalModulesSupport = false;
   };
 
   ClangdLSPServer(Transport &Transp, const ThreadsafeFS &TFS,

clang-tools-extra/clangd/ClangdServer.cpp

@@ -213,10 +213,11 @@ ClangdServer::Options::operator TUScheduler::Options() const {
 
 ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
                            const ThreadsafeFS &TFS, const Options &Opts,
-                           Callbacks *Callbacks, ModulesBuilder *ModulesManager)
+                           Callbacks *Callbacks)
     : FeatureModules(Opts.FeatureModules), CDB(CDB), TFS(TFS),
       DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr),
-      ModulesManager(ModulesManager), ClangTidyProvider(Opts.ClangTidyProvider),
+      ModulesManager(Opts.ModulesManager),
+      ClangTidyProvider(Opts.ClangTidyProvider),
       UseDirtyHeaders(Opts.UseDirtyHeaders),
       LineFoldingOnly(Opts.LineFoldingOnly),
       PreambleParseForwardingFunctions(Opts.PreambleParseForwardingFunctions),
@@ -228,7 +229,6 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
       DirtyFS(std::make_unique<DraftStoreFS>(TFS, DraftMgr)) {
   if (Opts.AsyncThreadsCount != 0)
     IndexTasks.emplace();
-
   // Pass a callback into `WorkScheduler` to extract symbols from a newly
   // parsed file and rebuild the file index synchronously each time an AST
   // is parsed.

clang-tools-extra/clangd/ClangdServer.h

@@ -43,7 +43,6 @@
 
 namespace clang {
 namespace clangd {
-
 /// Manages a collection of source files and derived data (ASTs, indexes),
 /// and provides language-aware features such as code completion.
 ///
@@ -114,8 +113,8 @@ class ClangdServer {
     /// This throttler controls which preambles may be built at a given time.
     clangd::PreambleThrottler *PreambleThrottler = nullptr;
 
-    /// Enable experimental support for modules.
-    bool ExperimentalModulesSupport = false;
+    /// Manages to build module files.
+    ModulesBuilder *ModulesManager = nullptr;
 
     /// If true, ClangdServer builds a dynamic in-memory index for symbols in
     /// opened files and uses the index to augment code completion results.
@@ -204,8 +203,7 @@ class ClangdServer {
   /// those arguments for subsequent reparses. However, ClangdServer will check
   /// if compilation arguments changed on calls to forceReparse().
   ClangdServer(const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS,
-               const Options &Opts, Callbacks *Callbacks = nullptr,
-               ModulesBuilder *ModulesManager = nullptr);
+               const Options &Opts, Callbacks *Callbacks = nullptr);
   ~ClangdServer();
 
   /// Gets the installed feature module of a given type, if any.

clang-tools-extra/clangd/ModulesBuilder.cpp

@@ -61,19 +61,6 @@ llvm::SmallString<256> getUniqueModuleFilesPath(PathRef MainFile) {
   return Result;
 }
 
-// Get the absolute path for the filename from the compile command.
-llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
-  llvm::SmallString<128> AbsolutePath;
-  if (llvm::sys::path::is_absolute(Cmd.Filename)) {
-    AbsolutePath = Cmd.Filename;
-  } else {
-    AbsolutePath = Cmd.Directory;
-    llvm::sys::path::append(AbsolutePath, Cmd.Filename);
-    llvm::sys::path::remove_dots(AbsolutePath, true);
-  }
-  return AbsolutePath;
-}
-
 // Get a unique module file path under \param ModuleFilesPrefix.
 std::string getModuleFilePath(llvm::StringRef ModuleName,
                               PathRef ModuleFilesPrefix) {
@@ -181,19 +168,17 @@ class StandalonePrerequisiteModules : public PrerequisiteModules {
   llvm::StringSet<> BuiltModuleNames;
 };
 
-// Build a module file for module with `ModuleName`. Return false
-// when there are problem happens. Return true when the
-// module file exists or built successfully. The information of built
+// Build a module file for module with `ModuleName`. The information of built
 // module file are stored in \param BuiltModuleFiles.
 llvm::Error buildModuleFile(llvm::StringRef ModuleName,
                             const GlobalCompilationDatabase &CDB,
-                            const ThreadsafeFS &TFS, ProjectModules *MDB,
+                            const ThreadsafeFS &TFS, ProjectModules &MDB,
                             PathRef ModuleFilesPrefix,
                             StandalonePrerequisiteModules &BuiltModuleFiles) {
   if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
     return llvm::Error::success();
 
-  PathRef ModuleUnitFileName = MDB->getSourceForModuleName(ModuleName);
+  PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName);
   // It is possible that we're meeting third party modules (modules whose
   // source are not in the project. e.g, the std module may be a third-party
   // module for most projects) or something wrong with the implementation of
@@ -205,19 +190,22 @@ llvm::Error buildModuleFile(llvm::StringRef ModuleName,
     return llvm::createStringError(llvm::formatv(
         "Failed to build '{0}': Failed to get the primary source", ModuleName));
 
-  for (auto &RequiredModuleName : MDB->getRequiredModules(ModuleUnitFileName)) {
-    // Return early if there are errors building the module file.
-    if (llvm::Error Err = buildModuleFile(RequiredModuleName, CDB, TFS, MDB,
-                                          ModuleFilesPrefix, BuiltModuleFiles))
-      return Err;
-  }
-
+  // Try cheap operation earlier to boil-out cheaply if there are problems.
   auto Cmd = CDB.getCompileCommand(ModuleUnitFileName);
   if (!Cmd)
     return llvm::createStringError(
         llvm::formatv("Failed to build '{0}': No compile command for {1}",
                       ModuleName, ModuleUnitFileName));
 
+  for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName)) {
+    // Return early if there are errors building the module file.
+    if (llvm::Error Err = buildModuleFile(RequiredModuleName, CDB, TFS, MDB,
+                                          ModuleFilesPrefix, BuiltModuleFiles))
+      return llvm::createStringError(
+          llvm::formatv("Failed to build dependency {0}: {1}",
+                        RequiredModuleName, toString(std::move(Err))));
+  }
+
   Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix);
 
   ParseInputs Inputs;
@@ -271,12 +259,14 @@ std::unique_ptr<PrerequisiteModules>
 ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
                                             const ThreadsafeFS &TFS) const {
   std::unique_ptr<ProjectModules> MDB = CDB.getProjectModules(File);
-  if (!MDB)
-    return {};
+  if (!MDB) {
+    elog("Failed to get Project Modules information for {0}", File);
+    return std::make_unique<FailedPrerequisiteModules>();
+  }
 
   std::vector<std::string> RequiredModuleNames = MDB->getRequiredModules(File);
   if (RequiredModuleNames.empty())
-    return {};
+    return std::make_unique<StandalonePrerequisiteModules>();
 
   llvm::SmallString<256> ModuleFilesPrefix = getUniqueModuleFilesPath(File);
 
@@ -288,7 +278,7 @@ ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
   for (llvm::StringRef RequiredModuleName : RequiredModuleNames) {
     // Return early if there is any error.
     if (llvm::Error Err =
-            buildModuleFile(RequiredModuleName, CDB, TFS, MDB.get(),
+            buildModuleFile(RequiredModuleName, CDB, TFS, *MDB.get(),
                             ModuleFilesPrefix, *RequiredModules.get())) {
       elog("Failed to build module {0}; due to {1}", RequiredModuleName,
            toString(std::move(Err)));
@@ -304,6 +294,9 @@ ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
 bool StandalonePrerequisiteModules::canReuse(
     const CompilerInvocation &CI,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
+  if (RequiredModules.empty())
+    return true;
+
   CompilerInstance Clang;
 
   Clang.setInvocation(std::make_shared<CompilerInvocation>(CI));

clang-tools-extra/clangd/Preamble.cpp

@@ -587,10 +587,11 @@ class DiagPatcher {
 };
 } // namespace
 
-std::shared_ptr<const PreambleData> buildPreamble(
-    PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs,
-    bool StoreInMemory, PreambleParsedCallback PreambleCallback,
-    PreambleBuildStats *Stats) {
+std::shared_ptr<const PreambleData>
+buildPreamble(PathRef FileName, CompilerInvocation CI,
+              const ParseInputs &Inputs, bool StoreInMemory,
+              PreambleParsedCallback PreambleCallback,
+              PreambleBuildStats *Stats) {
   // Note that we don't need to copy the input contents, preamble can live
   // without those.
   auto ContentsBuffer =
@@ -659,7 +660,6 @@ std::shared_ptr<const PreambleData> buildPreamble(
 
   WallTimer PreambleTimer;
   PreambleTimer.startTimer();
-
   auto BuiltPreamble = PrecompiledPreamble::Build(
       CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
       Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(),

clang-tools-extra/clangd/Preamble.h

@@ -27,7 +27,6 @@
 #include "Diagnostics.h"
 #include "FS.h"
 #include "Headers.h"
-
 #include "ModulesBuilder.h"
 
 #include "clang-include-cleaner/Record.h"

clang-tools-extra/clangd/ScanningProjectModules.h

@@ -15,7 +15,7 @@
 namespace clang {
 namespace clangd {
 
-// Providing modules information for the project by scanning every file.
+/// Providing modules information for the project by scanning every file.
 std::unique_ptr<ProjectModules> scanningProjectModules(
     std::shared_ptr<const clang::tooling::CompilationDatabase> CDB,
     const ThreadsafeFS &TFS);

clang-tools-extra/clangd/TUScheduler.cpp

@@ -710,7 +710,6 @@ class ASTWorker {
   TUScheduler::ASTCache &IdleASTs;
   TUScheduler::HeaderIncluderCache &HeaderIncluders;
   const bool RunSync;
-
   /// Time to wait after an update to see whether another update obsoletes it.
   const DebouncePolicy UpdateDebounce;
   /// File that ASTWorker is responsible for.
@@ -1083,7 +1082,6 @@ void PreambleThread::build(Request Req) {
 
   PreambleBuildStats Stats;
   bool IsFirstPreamble = !LatestBuild;
-
   LatestBuild = clang::clangd::buildPreamble(
       FileName, *Req.CI, Inputs, StoreInMemory,
       [&](CapturedASTCtx ASTCtx,

clang-tools-extra/clangd/tool/Check.cpp

@@ -146,8 +146,6 @@ class Checker {
   ClangdLSPServer::Options Opts;
   // from buildCommand
   tooling::CompileCommand Cmd;
-  std::unique_ptr<GlobalCompilationDatabase> BaseCDB;
-  std::unique_ptr<OverlayCDB> CDB;
   // from buildInvocation
   ParseInputs Inputs;
   std::unique_ptr<CompilerInvocation> Invocation;
@@ -170,14 +168,14 @@ class Checker {
     DirectoryBasedGlobalCompilationDatabase::Options CDBOpts(TFS);
     CDBOpts.CompileCommandsDir =
         Config::current().CompileFlags.CDBSearch.FixedCDBPath;
-    BaseCDB =
+    std::unique_ptr<GlobalCompilationDatabase> BaseCDB =
         std::make_unique<DirectoryBasedGlobalCompilationDatabase>(CDBOpts);
     auto Mangler = CommandMangler::detect();
     Mangler.SystemIncludeExtractor =
         getSystemIncludeExtractor(llvm::ArrayRef(Opts.QueryDriverGlobs));
     if (Opts.ResourceDir)
       Mangler.ResourceDir = *Opts.ResourceDir;
-    CDB = std::make_unique<OverlayCDB>(
+    auto CDB = std::make_unique<OverlayCDB>(
         BaseCDB.get(), std::vector<std::string>{}, std::move(Mangler));
 
     if (auto TrueCmd = CDB->getCompileCommand(File)) {
@@ -236,7 +234,6 @@ class Checker {
   // Build preamble and AST, and index them.
   bool buildAST() {
     log("Building preamble...");
-
     Preamble = buildPreamble(
         File, *Invocation, Inputs, /*StoreInMemory=*/true,
         [&](CapturedASTCtx Ctx,

clang-tools-extra/clangd/tool/ClangdMain.cpp

@@ -869,7 +869,7 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
 
   ClangdLSPServer::Options Opts;
   Opts.UseDirBasedCDB = (CompileArgsFrom == FilesystemCompileArgs);
-  Opts.ExperimentalModulesSupport = ExperimentalModulesSupport;
+  Opts.EnableExperimentalModulesSupport = ExperimentalModulesSupport;
 
   switch (PCHStorage) {
   case PCHStorageFlag::Memory:

clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp

@@ -132,11 +132,8 @@ CodeCompleteResult completions(const TestTU &TU, Position Point,
     ADD_FAILURE() << "Couldn't build CompilerInvocation";
     return {};
   }
-
-  MockCompilationDatabase CDB;
   auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
-                                /*InMemory=*/true,
-                                /*Callback=*/nullptr);
+                                /*InMemory=*/true, /*Callback=*/nullptr);
   return codeComplete(testPath(TU.Filename), Point, Preamble.get(), Inputs,
                       Opts);
 }
@@ -1366,12 +1363,8 @@ signatures(llvm::StringRef Text, Position Point,
     ADD_FAILURE() << "Couldn't build CompilerInvocation";
     return {};
   }
-
-  MockCompilationDatabase CDB;
-  auto Preamble =
-      buildPreamble(testPath(TU.Filename), *CI, Inputs,
-                    /*InMemory=*/true,
-                    /*Callback=*/nullptr);
+  auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
+                                /*InMemory=*/true, /*Callback=*/nullptr);
   if (!Preamble) {
     ADD_FAILURE() << "Couldn't build Preamble";
     return {};
@@ -1669,12 +1662,8 @@ TEST(SignatureHelpTest, StalePreamble) {
   auto Inputs = TU.inputs(FS);
   auto CI = buildCompilerInvocation(Inputs, Diags);
   ASSERT_TRUE(CI);
-
-  MockCompilationDatabase CDB;
-  auto EmptyPreamble =
-      buildPreamble(testPath(TU.Filename), *CI, Inputs,
-                    /*InMemory=*/true,
-                    /*Callback=*/nullptr);
+  auto EmptyPreamble = buildPreamble(testPath(TU.Filename), *CI, Inputs,
+                                     /*InMemory=*/true, /*Callback=*/nullptr);
   ASSERT_TRUE(EmptyPreamble);
 
   TU.AdditionalFiles["a.h"] = "int foo(int x);";

clang-tools-extra/clangd/unittests/FileIndexTests.cpp

@@ -336,8 +336,6 @@ TEST(FileIndexTest, RebuildWithPreamble) {
 
   FileIndex Index;
   bool IndexUpdated = false;
-
-  MockCompilationDatabase CDB;
   buildPreamble(
       FooCpp, *CI, PI,
       /*StoreInMemory=*/true,

clang-tools-extra/clangd/unittests/ParsedASTTests.cpp

@@ -376,7 +376,6 @@ TEST(ParsedASTTest, PatchesAdditionalIncludes) {
   MockFS FS;
   auto Inputs = TU.inputs(FS);
   auto CI = buildCompilerInvocation(Inputs, Diags);
-  MockCompilationDatabase CDB;
   auto EmptyPreamble =
       buildPreamble(testPath("foo.cpp"), *CI, Inputs, true, nullptr);
   ASSERT_TRUE(EmptyPreamble);
@@ -419,7 +418,6 @@ TEST(ParsedASTTest, PatchesDeletedIncludes) {
   MockFS FS;
   auto Inputs = TU.inputs(FS);
   auto CI = buildCompilerInvocation(Inputs, Diags);
-  MockCompilationDatabase CDB;
   auto BaselinePreamble =
       buildPreamble(testPath("foo.cpp"), *CI, Inputs, true, nullptr);
   ASSERT_TRUE(BaselinePreamble);

clang-tools-extra/clangd/unittests/PreambleTests.cpp

@@ -192,10 +192,8 @@ TEST(PreamblePatchTest, PatchesPreambleIncludes) {
   TU.AdditionalFiles["b.h"] = "";
   TU.AdditionalFiles["c.h"] = "";
   auto PI = TU.inputs(FS);
-  MockCompilationDatabase CDB;
-  auto BaselinePreamble =
-      buildPreamble(TU.Filename, *buildCompilerInvocation(PI, Diags), PI, true,
-                    nullptr);
+  auto BaselinePreamble = buildPreamble(
+      TU.Filename, *buildCompilerInvocation(PI, Diags), PI, true, nullptr);
   // We drop c.h from modified and add a new header. Since the latter is patched
   // we should only get a.h in preamble includes. d.h shouldn't be part of the
   // preamble, as it's coming from a disabled region.