[clang][clang-scan-deps] Aggregate the full dependency information.

Differential Revision: https://reviews.llvm.org/D70268

This is a recommit of f978ea4 with a fix for the PowerPC failure.

The issue was that:
* `CompilerInstance::ExecuteAction` calls
  `getTarget().adjust(getLangOpts());`.
* `PPCTargetInfo::adjust` changes `LangOptions::HasAltivec`.
* This happens after the first few calls to `getModuleHash`.

There’s even a FIXME saying:
```
  // FIXME: We shouldn't need to do this, the target should be immutable once
  // created. This complexity should be lifted elsewhere.
```

This only showed up on PowerPC because it's one of the few targets that
almost always changes a hashed langopt.

I looked into addressing the fixme, but that would be a much larger
change, and it's not the only thing that happens in `ExecuteAction` that
can change the module context hash. Instead I changed the code to not
call `getModuleHash` until after it has been modified in `ExecuteAction`.

(cherry picked from commit 356a4b4)

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -11,13 +11,69 @@
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Tooling/JSONCompilationDatabase.h"
+#include "llvm/ADT/StringSet.h"
 #include <string>
 
 namespace clang{
 namespace tooling{
 namespace dependencies{
 
+/// The full dependencies and module graph for a specific input.
+struct FullDependencies {
+  /// The name of the C++20 module this translation unit exports. This may
+  /// include `:` for C++20 module partitons.
+  ///
+  /// If the translation unit is not a module then this will be empty.
+  std::string ExportedModuleName;
+
+  /// The context hash represents the set of compiler options that may make one
+  /// version of a module incompatible with another. This includes things like
+  /// language mode, predefined macros, header search paths, etc...
+  ///
+  /// Modules with the same name but a different \c ContextHash should be
+  /// treated as separate modules for the purpose of a build.
+  std::string ContextHash;
+
+  /// A collection of absolute paths to files that this translation unit
+  /// directly depends on, not including transitive dependencies.
+  std::vector<std::string> FileDeps;
+
+  /// A list of modules this translation unit directly depends on, not including
+  /// transitive dependencies.
+  ///
+  /// This may include modules with a different context hash when it can be
+  /// determined that the differences are benign for this compilation.
+  std::vector<ClangModuleDep> ClangModuleDeps;
+
+  /// A partial addtional set of command line arguments that can be used to
+  /// build this translation unit.
+  ///
+  /// Call \c getFullAdditionalCommandLine() to get a command line suitable for
+  /// appending to the original command line to pass to clang.
+  std::vector<std::string> AdditionalNonPathCommandLine;
+
+  /// Gets the full addtional command line suitable for appending to the
+  /// original command line to pass to clang.
+  ///
+  /// \param LookupPCMPath this function is called to fill in `-fmodule-file=`
+  ///                      flags and for the `-o` flag. It needs to return a
+  ///                      path for where the PCM for the given module is to
+  ///                      be located.
+  /// \param LookupModuleDeps this fucntion is called to collect the full
+  ///                         transitive set of dependencies for this
+  ///                         compilation.
+  std::vector<std::string> getAdditionalCommandLine(
+      std::function<StringRef(ClangModuleDep)> LookupPCMPath,
+      std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const;
+};
+
+struct FullDependenciesResult {
+  FullDependencies FullDeps;
+  std::vector<ModuleDeps> DiscoveredModules;
+};
+
 /// The high-level implementation of the dependency discovery tool that runs on
 /// an individual worker thread.
 class DependencyScanningTool {
@@ -35,8 +91,23 @@ class DependencyScanningTool {
   getDependencyFile(const tooling::CompilationDatabase &Compilations,
                     StringRef CWD);
 
+  /// Collect the full module depenedency graph for the input, ignoring any
+  /// modules which have already been seen.
+  ///
+  /// \param AlreadySeen this is used to not report modules that have previously
+  ///                    been reported. Use the same `llvm::StringSet<>` for all
+  ///                    calls to `getFullDependencies` for a single
+  ///                    `DependencyScanningTool` for a single build. Use a
+  ///                    different one for different tools, and clear it between
+  ///                    builds.
+  ///
+  /// \returns a \c StringError with the diagnostic output if clang errors
+  /// occurred, \c FullDependencies otherwise.
+  llvm::Expected<FullDependenciesResult>
+  getFullDependencies(const tooling::CompilationDatabase &Compilations,
+                      StringRef CWD, const llvm::StringSet<> &AlreadySeen);
+
 private:
-  const ScanningOutputFormat Format;
   DependencyScanningWorker Worker;
 };
 

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -28,16 +28,82 @@ namespace dependencies {
 
 class DependencyConsumer;
 
+/// This is used to refer to a specific module.
+///
+/// See \c ModuleDeps for details about what these members mean.
+struct ClangModuleDep {
+  std::string ModuleName;
+  std::string ContextHash;
+};
+
 struct ModuleDeps {
+  /// The name of the module. This may include `:` for C++20 module partitons,
+  /// or a header-name for C++20 header units.
   std::string ModuleName;
-  std::string ClangModuleMapFile;
-  std::string ModulePCMPath;
+
+  /// The context hash of a module represents the set of compiler options that
+  /// may make one version of a module incompatible with another. This includes
+  /// things like language mode, predefined macros, header search paths, etc...
+  ///
+  /// Modules with the same name but a different \c ContextHash should be
+  /// treated as separate modules for the purpose of a build.
   std::string ContextHash;
+
+  /// The path to the modulemap file which defines this module.
+  ///
+  /// This can be used to explicitly build this module. This file will
+  /// additionally appear in \c FileDeps as a dependency.
+  std::string ClangModuleMapFile;
+
+  /// The path to where an implicit build would put the PCM for this module.
+  std::string ImplicitModulePCMPath;
+
+  /// A collection of absolute paths to files that this module directly depends
+  /// on, not including transitive dependencies.
   llvm::StringSet<> FileDeps;
-  llvm::StringSet<> ClangModuleDeps;
+
+  /// A list of modules this module directly depends on, not including
+  /// transitive dependencies.
+  ///
+  /// This may include modules with a different context hash when it can be
+  /// determined that the differences are benign for this compilation.
+  std::vector<ClangModuleDep> ClangModuleDeps;
+
+  /// A partial command line that can be used to build this module.
+  ///
+  /// Call \c getFullCommandLine() to get a command line suitable for passing to
+  /// clang.
+  std::vector<std::string> NonPathCommandLine;
+
+  // Used to track which modules that were discovered were directly imported by
+  // the primary TU.
   bool ImportedByMainFile = false;
+
+  /// Gets the full command line suitable for passing to clang.
+  ///
+  /// \param LookupPCMPath this function is called to fill in `-fmodule-file=`
+  ///                      flags and for the `-o` flag. It needs to return a
+  ///                      path for where the PCM for the given module is to
+  ///                      be located.
+  /// \param LookupModuleDeps this fucntion is called to collect the full
+  ///                         transitive set of dependencies for this
+  ///                         compilation.
+  std::vector<std::string> getFullCommandLine(
+      std::function<StringRef(ClangModuleDep)> LookupPCMPath,
+      std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const;
 };
 
+namespace detail {
+/// Append the `-fmodule-file=` and `-fmodule-map-file=` arguments for the
+/// modules in \c Modules transitively, along with other needed arguments to
+/// use explicitly built modules.
+void appendCommonModuleArguments(
+    llvm::ArrayRef<ClangModuleDep> Modules,
+    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
+    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps,
+    std::vector<std::string> &Result);
+} // namespace detail
+
 class ModuleDepCollector;
 
 class ModuleDepCollectorPP final : public PPCallbacks {
@@ -54,6 +120,8 @@ class ModuleDepCollectorPP final : public PPCallbacks {
                           StringRef SearchPath, StringRef RelativePath,
                           const Module *Imported,
                           SrcMgr::CharacteristicKind FileType) override;
+  void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
+                    const Module *Imported) override;
 
   void EndOfMainFile() override;
 
@@ -62,16 +130,18 @@ class ModuleDepCollectorPP final : public PPCallbacks {
   ModuleDepCollector &MDC;
   llvm::DenseSet<const Module *> DirectDeps;
 
+  void handleImport(const Module *Imported);
   void handleTopLevelModule(const Module *M);
-  void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD);
-  void addModuleDep(const Module *M, ModuleDeps &MD);
-
-  void addDirectDependencies(const Module *Mod);
+  void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD,
+                           llvm::DenseSet<const Module *> &AddedModules);
+  void addModuleDep(const Module *M, ModuleDeps &MD,
+                    llvm::DenseSet<const Module *> &AddedModules);
 };
 
 class ModuleDepCollector final : public DependencyCollector {
 public:
-  ModuleDepCollector(CompilerInstance &I, DependencyConsumer &C);
+  ModuleDepCollector(std::unique_ptr<DependencyOutputOptions> Opts,
+                     CompilerInstance &I, DependencyConsumer &C);
 
   void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
@@ -85,6 +155,7 @@ class ModuleDepCollector final : public DependencyCollector {
   std::string ContextHash;
   std::vector<std::string> MainDeps;
   std::unordered_map<std::string, ModuleDeps> Deps;
+  std::unique_ptr<DependencyOutputOptions> Opts;
 };
 
 } // end namespace dependencies