[C++20] [Modules] [ClangScanDeps] Add ClangScanDeps support for C++20…

… Named Modules in P1689 format (2/4)

Close #51792
Close #56770

This patch adds ClangScanDeps support for C++20 Named Modules in P1689
format. We can find the P1689 format at:
https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html.
After we land the patch, we're able to compile C++20 Named
Modules with CMake! And although P1689 is written by kitware people,
other build systems should be able to use the format to compile C++20
Named Modules too.

TODO: Support header units in P1689 Format.
TODO2: Support C++20 Modules in the full dependency format of
ClangScanDeps. We also want to support C++20 Modules and clang modules
together according to
https://discourse.llvm.org/t/how-should-we-support-dependency-scanner-for-c-20-modules/66027.
But P1689 format cares about C++20 Modules only for now. So let's focus
on C++ Modules and P1689 format. And look at the full dependency format
later.

I'll add the ReleaseNotes and Documentations after the patch get landed.

Reviewed By: jansvoboda11

Differential Revision: https://reviews.llvm.org/D137527

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

@@ -35,9 +35,13 @@ enum class ScanningOutputFormat {
   /// intermodule dependency information.
   Make,
 
-  /// This outputs the full module dependency graph suitable for use for
+  /// This outputs the full clang module dependency graph suitable for use for
   /// explicitly building modules.
   Full,
+
+  /// This outputs the dependency graph for standard c++ modules in P1689R5
+  /// format.
+  P1689,
 };
 
 /// The dependency scanning service contains shared configuration and state that

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -68,6 +68,12 @@ struct TranslationUnitDeps {
   std::vector<std::string> DriverCommandLine;
 };
 
+struct P1689Rule {
+  std::string PrimaryOutput;
+  std::optional<P1689ModuleInfo> Provides;
+  std::vector<P1689ModuleInfo> Requires;
+};
+
 /// The high-level implementation of the dependency discovery tool that runs on
 /// an individual worker thread.
 class DependencyScanningTool {
@@ -86,6 +92,10 @@ class DependencyScanningTool {
   llvm::Expected<std::string>
   getDependencyFile(const std::vector<std::string> &CommandLine, StringRef CWD);
 
+  llvm::Expected<P1689Rule>
+  getP1689ModuleDependencyFile(const CompileCommand &Command,
+                               StringRef CWD);
+
   /// Given a Clang driver command-line for a translation unit, gather the
   /// modular dependencies and return the information needed for explicit build.
   ///

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -41,7 +41,11 @@ class DependencyConsumer {
 public:
   virtual ~DependencyConsumer() {}
 
-  virtual void handleBuildCommand(Command Cmd) = 0;
+  virtual void handleProvidedAndRequiredStdCXXModules(
+      std::optional<P1689ModuleInfo> Provided,
+      std::vector<P1689ModuleInfo> Requires) {}
+
+  virtual void handleBuildCommand(Command Cmd) {}
 
   virtual void
   handleDependencyOutputOpts(const DependencyOutputOptions &Opts) = 0;

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -62,6 +62,27 @@ struct ModuleID {
   }
 };
 
+/// P1689ModuleInfo - Represents the needed information of standard C++20
+/// modules for P1689 format.
+struct P1689ModuleInfo {
+  /// The name of the module. This may include `:` for partitions.
+  std::string ModuleName;
+
+  /// Optional. The source path to the module.
+  std::string SourcePath;
+
+  /// If this module is a standard c++ interface unit.
+  bool IsStdCXXModuleInterface = true;
+
+  enum class ModuleType {
+    NamedCXXModule
+    // To be supported
+    // AngleHeaderUnit,
+    // QuoteHeaderUnit
+  };
+  ModuleType Type = ModuleType::NamedCXXModule;
+};
+
 /// An output from a module compilation, such as the path of the module file.
 enum class ModuleOutputKind {
   /// The module file (.pcm). Required.
@@ -181,7 +202,7 @@ class ModuleDepCollector final : public DependencyCollector {
   ModuleDepCollector(std::unique_ptr<DependencyOutputOptions> Opts,
                      CompilerInstance &ScanInstance, DependencyConsumer &C,
                      CompilerInvocation OriginalCI, bool OptimizeArgs,
-                     bool EagerLoadModules);
+                     bool EagerLoadModules, bool IsStdModuleP1689Format);
 
   void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
@@ -219,6 +240,12 @@ class ModuleDepCollector final : public DependencyCollector {
   bool OptimizeArgs;
   /// Whether to set up command-lines to load PCM files eagerly.
   bool EagerLoadModules;
+  /// If we're generating dependency output in P1689 format
+  /// for standard C++ modules.
+  bool IsStdModuleP1689Format;
+
+  std::optional<P1689ModuleInfo> ProvidedStdCXXModule;
+  std::vector<P1689ModuleInfo> RequiredStdCXXModules;
 
   /// Checks whether the module is known as being prebuilt.
   bool isPrebuiltModule(const Module *M);

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -88,6 +88,49 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
   return Output;
 }
 
+llvm::Expected<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile(
+    const CompileCommand &Command, StringRef CWD) {
+  class P1689ModuleDependencyPrinterConsumer : public DependencyConsumer {
+  public:
+    P1689ModuleDependencyPrinterConsumer(P1689Rule &Rule,
+                                         const CompileCommand &Command)
+        : Filename(Command.Filename), Rule(Rule) {
+      Rule.PrimaryOutput = Command.Output;
+    }
+
+    void
+    handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {}
+    void handleFileDependency(StringRef File) override {}
+    void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {}
+    void handleModuleDependency(ModuleDeps MD) override {}
+    void handleContextHash(std::string Hash) override {}
+    std::string lookupModuleOutput(const ModuleID &ID,
+                                   ModuleOutputKind Kind) override {
+      llvm::report_fatal_error("unexpected call to lookupModuleOutput");
+    }
+
+    void handleProvidedAndRequiredStdCXXModules(
+        std::optional<P1689ModuleInfo> Provided,
+        std::vector<P1689ModuleInfo> Requires) override {
+      Rule.Provides = Provided;
+      if (Rule.Provides)
+        Rule.Provides->SourcePath = Filename.str();
+      Rule.Requires = Requires;
+    }
+
+  private:
+    StringRef Filename;
+    P1689Rule &Rule;
+  };
+
+  P1689Rule Rule;
+  P1689ModuleDependencyPrinterConsumer Consumer(Rule, Command);
+  auto Result = Worker.computeDependencies(CWD, Command.CommandLine, Consumer);
+  if (Result)
+    return std::move(Result);
+  return Rule;
+}
+
 llvm::Expected<TranslationUnitDeps>
 DependencyScanningTool::getTranslationUnitDependencies(
     const std::vector<std::string> &CommandLine, StringRef CWD,

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

@@ -247,10 +247,12 @@ class DependencyScanningAction : public tooling::ToolAction {
           std::make_shared<DependencyConsumerForwarder>(
               std::move(Opts), WorkingDirectory, Consumer));
       break;
+    case ScanningOutputFormat::P1689:
     case ScanningOutputFormat::Full:
       MDC = std::make_shared<ModuleDepCollector>(
           std::move(Opts), ScanInstance, Consumer, OriginalInvocation,
-          OptimizeArgs, EagerLoadModules);
+          OptimizeArgs, EagerLoadModules,
+          Format == ScanningOutputFormat::P1689);
       ScanInstance.addDependencyCollector(MDC);
       break;
     }

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -341,6 +341,14 @@ void ModuleDepCollectorPP::InclusionDirective(
 void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
                                         ModuleIdPath Path,
                                         const Module *Imported) {
+  if (MDC.ScanInstance.getPreprocessor().isInImportingCXXNamedModules()) {
+    P1689ModuleInfo RequiredModule;
+    RequiredModule.ModuleName = Path[0].first->getName().str();
+    RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
+    MDC.RequiredStdCXXModules.push_back(RequiredModule);
+    return;
+  }
+
   handleImport(Imported);
 }
 
@@ -363,6 +371,21 @@ void ModuleDepCollectorPP::EndOfMainFile() {
                                  .getFileEntryForID(MainFileID)
                                  ->getName());
 
+  auto &PP = MDC.ScanInstance.getPreprocessor();
+  if (PP.isInNamedModule()) {
+    P1689ModuleInfo ProvidedModule;
+    ProvidedModule.ModuleName = PP.getNamedModuleName();
+    ProvidedModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
+    ProvidedModule.IsStdCXXModuleInterface = PP.isInNamedInterfaceUnit();
+    // Don't put implementation (non partition) unit as Provide.
+    // Put the module as required instead. Since the implementation
+    // unit will import the primary module implicitly.
+    if (PP.isInImplementationUnit())
+      MDC.RequiredStdCXXModules.push_back(ProvidedModule);
+    else
+      MDC.ProvidedStdCXXModule = ProvidedModule;
+  }
+
   if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
     MDC.addFileDep(MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);
 
@@ -376,6 +399,10 @@ void ModuleDepCollectorPP::EndOfMainFile() {
 
   MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);
 
+  if (MDC.IsStdModuleP1689Format)
+    MDC.Consumer.handleProvidedAndRequiredStdCXXModules(
+        MDC.ProvidedStdCXXModule, MDC.RequiredStdCXXModules);
+
   for (auto &&I : MDC.ModularDeps)
     MDC.Consumer.handleModuleDependency(*I.second);
 
@@ -550,10 +577,12 @@ void ModuleDepCollectorPP::addAffectingClangModule(
 ModuleDepCollector::ModuleDepCollector(
     std::unique_ptr<DependencyOutputOptions> Opts,
     CompilerInstance &ScanInstance, DependencyConsumer &C,
-    CompilerInvocation OriginalCI, bool OptimizeArgs, bool EagerLoadModules)
+    CompilerInvocation OriginalCI, bool OptimizeArgs, bool EagerLoadModules,
+    bool IsStdModuleP1689Format)
     : ScanInstance(ScanInstance), Consumer(C), Opts(std::move(Opts)),
       OriginalInvocation(std::move(OriginalCI)), OptimizeArgs(OptimizeArgs),
-      EagerLoadModules(EagerLoadModules) {}
+      EagerLoadModules(EagerLoadModules),
+      IsStdModuleP1689Format(IsStdModuleP1689Format) {}
 
 void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
   PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(*this));

clang/test/ClangScanDeps/P1689.cppm

@@ -0,0 +1,157 @@
+// RUN: rm -fr %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: sed "s|DIR|%/t|g" %t/P1689.json.in > %t/P1689.json
+// RUN: clang-scan-deps -compilation-database %t/P1689.json -format=p1689 | FileCheck %t/Checks.cpp -DPREFIX=%/t
+// RUN: clang-scan-deps --mode=preprocess-dependency-directives -compilation-database %t/P1689.json -format=p1689 | FileCheck %t/Checks.cpp -DPREFIX=%/t
+
+//--- P1689.json.in
+[
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/M.cppm -c -o DIR/M.o",
+  "file": "DIR/M.cppm",
+  "output": "DIR/M.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/Impl.cpp -c -o DIR/Impl.o",
+  "file": "DIR/Impl.cpp",
+  "output": "DIR/Impl.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/impl_part.cppm -c -o DIR/impl_part.o",
+  "file": "DIR/impl_part.cppm",
+  "output": "DIR/impl_part.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/interface_part.cppm -c -o DIR/interface_part.o",
+  "file": "DIR/interface_part.cppm",
+  "output": "DIR/interface_part.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/User.cpp -c -o DIR/User.o",
+  "file": "DIR/User.cpp",
+  "output": "DIR/User.o"
+}
+]
+
+
+//--- M.cppm
+export module M;
+export import :interface_part;
+import :impl_part;
+export void Hello();
+
+//--- Impl.cpp
+module;
+#include "header.mock"
+module M;
+void Hello() {
+    std::cout << "Hello ";
+}
+
+//--- impl_part.cppm
+module;
+#include "header.mock"
+module M:impl_part;
+import :interface_part;
+
+std::string W = "World.";
+void World() {
+    std::cout << W << std::endl;
+}
+
+//--- interface_part.cppm
+export module M:interface_part;
+export void World();
+
+//--- User.cpp
+import M;
+import third_party_module;
+int main() {
+    Hello();
+    World();
+    return 0;
+}
+
+//--- Checks.cpp
+// CHECK: {
+// CHECK-NEXT:   "revision": 0,
+// CHECK-NEXT:   "rules": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "primary-output": "[[PREFIX]]/Impl.o",
+// CHECK-NEXT:       "requires": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "logical-name": "M",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/M.cppm"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "primary-output": "[[PREFIX]]/M.o",
+// CHECK-NEXT:       "provides": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "is-interface": true,
+// CHECK-NEXT:           "logical-name": "M",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/M.cppm"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "requires": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "logical-name": "M:interface_part",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/interface_part.cppm"
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "logical-name": "M:impl_part",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/impl_part.cppm"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "primary-output": "[[PREFIX]]/User.o",
+// CHECK-NEXT:       "requires": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "logical-name": "M",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/M.cppm"
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "logical-name": "third_party_module"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "primary-output": "[[PREFIX]]/impl_part.o",
+// CHECK-NEXT:       "provides": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "is-interface": false,
+// CHECK-NEXT:           "logical-name": "M:impl_part",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/impl_part.cppm"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ],
+// CHECK-NEXT:       "requires": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "logical-name": "M:interface_part",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/interface_part.cppm"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "primary-output": "[[PREFIX]]/interface_part.o",
+// CHECK-NEXT:       "provides": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:           "is-interface": true,
+// CHECK-NEXT:           "logical-name": "M:interface_part",
+// CHECK-NEXT:           "source-path": "[[PREFIX]]/interface_part.cppm"
+// CHECK-NEXT:         }
+// CHECK-NEXT:       ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ],
+// CHECK-NEXT:   "version": 1
+// CHECK-NEXT: }
+
+//--- header.mock