[clang][modules] Track affecting modules

When compiling a module, its semantics and Clang's behavior are affected by other modules. These modules are typically the **imported** ones. However, during implicit build, some modules end up being compiled and read without being actually imported. This patch starts tracking such modules and serializing them into `.pcm` files. This enables the dependency scanner to construct explicit compilations that mimic implicit build.

Reviewed By: benlangmuir

Differential Revision: https://reviews.llvm.org/D132430

clang/include/clang/Basic/Module.h

@@ -345,6 +345,10 @@ class Module {
   /// module depends.
   llvm::SmallSetVector<Module *, 2> Imports;
 
+  /// The set of top-level modules that affected the compilation of this module,
+  /// but were not imported.
+  llvm::SmallSetVector<Module *, 2> AffectingModules;
+
   /// Describes an exported module.
   ///
   /// The pointer is the module being re-exported, while the bit will be true

clang/include/clang/Lex/ModuleLoader.h

@@ -51,6 +51,11 @@ class ModuleLoadResult {
   ModuleLoadResult() = default;
   ModuleLoadResult(Module *M) : Storage(M, Normal) {}
   ModuleLoadResult(LoadResultKind Kind) : Storage(nullptr, Kind) {}
+  ModuleLoadResult(Module *M, LoadResultKind Kind) : Storage(M, Kind) {}
+
+  operator bool() const {
+    return Storage.getInt() == Normal && Storage.getPointer();
+  }
 
   operator Module *() const { return Storage.getPointer(); }
 

clang/include/clang/Lex/Preprocessor.h

@@ -860,6 +860,10 @@ class Preprocessor {
   /// The files that have been included.
   IncludedFilesSet IncludedFiles;
 
+  /// The set of top-level modules that affected preprocessing, but were not
+  /// imported.
+  llvm::SmallSetVector<Module *, 2> AffectingModules;
+
   /// The set of known macros exported from modules.
   llvm::FoldingSet<ModuleMacro> ModuleMacros;
 
@@ -1331,6 +1335,12 @@ class Preprocessor {
 
   /// \}
 
+  /// Get the set of top-level modules that affected preprocessing, but were not
+  /// imported.
+  const llvm::SmallSetVector<Module *, 2> &getAffectingModules() const {
+    return AffectingModules;
+  }
+
   /// Mark the file as included.
   /// Returns true if this is the first time the file was included.
   bool markIncluded(const FileEntry *File) {

clang/include/clang/Serialization/ASTBitCodes.h

@@ -829,6 +829,9 @@ enum SubmoduleRecordTypes {
   /// Specifies the name of the module that will eventually
   /// re-export the entities in this module.
   SUBMODULE_EXPORT_AS = 17,
+
+  /// Specifies affecting modules that were not imported.
+  SUBMODULE_AFFECTING_MODULES = 18,
 };
 
 /// Record types used within a comments block.

clang/include/clang/Serialization/ASTReader.h

@@ -689,7 +689,7 @@ class ASTReader
     Module *Mod;
 
     /// The kind of module reference.
-    enum { Import, Export, Conflict } Kind;
+    enum { Import, Export, Conflict, Affecting } Kind;
 
     /// The local ID of the module that is being exported.
     unsigned ID;

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -176,6 +176,13 @@ class ModuleDepCollectorPP final : public PPCallbacks {
                            llvm::DenseSet<const Module *> &AddedModules);
   void addModuleDep(const Module *M, ModuleDeps &MD,
                     llvm::DenseSet<const Module *> &AddedModules);
+
+  /// Traverses the affecting modules and updates \c MD with references to the
+  /// parent \c ModuleDepCollector info.
+  void addAllAffectingModules(const Module *M, ModuleDeps &MD,
+                              llvm::DenseSet<const Module *> &AddedModules);
+  void addAffectingModule(const Module *M, ModuleDeps &MD,
+                          llvm::DenseSet<const Module *> &AddedModules);
 };
 
 /// Collects modular and non-modular dependencies of the main file by attaching

clang/lib/Frontend/CompilerInstance.cpp

@@ -2099,7 +2099,7 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
         << Module->getFullModuleName()
         << SourceRange(Path.front().second, Path.back().second);
 
-      return ModuleLoadResult::MissingExpected;
+      return ModuleLoadResult(Module, ModuleLoadResult::MissingExpected);
     }
 
     // Check whether this module is available.

clang/lib/Lex/HeaderSearch.cpp

@@ -1563,6 +1563,8 @@ static bool suggestModule(HeaderSearch &HS, const FileEntry *File,
           *SuggestedModule = ModuleMap::KnownHeader();
         return true;
       }
+      // TODO: Add this module (or just its module map file) into something like
+      // `RequestingModule->AffectingModules`.
       return false;
     }
   }

clang/lib/Lex/PPDirectives.cpp

@@ -2281,6 +2281,13 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
     if (Imported) {
       Action = Import;
     } else if (Imported.isMissingExpected()) {
+      Module *M = static_cast<Module *>(Imported)->getTopLevelModule();
+      if (!BuildingSubmoduleStack.empty()) {
+        if (Imported != BuildingSubmoduleStack.back().M)
+          BuildingSubmoduleStack.back().M->AffectingModules.insert(M);
+      } else {
+        AffectingModules.insert(M);
+      }
       // We failed to find a submodule that we assumed would exist (because it
       // was in the directory of an umbrella header, for instance), but no
       // actual module containing it exists (because the umbrella header is

clang/lib/Serialization/ASTReader.cpp

@@ -4376,6 +4376,11 @@ ASTReader::ASTReadResult ASTReader::ReadAST(StringRef FileName,
         Unresolved.Mod->Imports.insert(ResolvedMod);
       continue;
 
+    case UnresolvedModuleRef::Affecting:
+      if (ResolvedMod)
+        Unresolved.Mod->AffectingModules.insert(ResolvedMod);
+      continue;
+
     case UnresolvedModuleRef::Export:
       if (ResolvedMod || Unresolved.IsWildcard)
         Unresolved.Mod->Exports.push_back(
@@ -5674,6 +5679,18 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
       }
       break;
 
+    case SUBMODULE_AFFECTING_MODULES:
+      for (unsigned Idx = 0; Idx != Record.size(); ++Idx) {
+        UnresolvedModuleRef Unresolved;
+        Unresolved.File = &F;
+        Unresolved.Mod = CurrentModule;
+        Unresolved.ID = Record[Idx];
+        Unresolved.Kind = UnresolvedModuleRef::Affecting;
+        Unresolved.IsWildcard = false;
+        UnresolvedModuleRefs.push_back(Unresolved);
+      }
+      break;
+
     case SUBMODULE_EXPORTS:
       for (unsigned Idx = 0; Idx + 1 < Record.size(); Idx += 2) {
         UnresolvedModuleRef Unresolved;

clang/lib/Serialization/ASTWriter.cpp

@@ -883,6 +883,7 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(SUBMODULE_TOPHEADER);
   RECORD(SUBMODULE_UMBRELLA_DIR);
   RECORD(SUBMODULE_IMPORTS);
+  RECORD(SUBMODULE_AFFECTING_MODULES);
   RECORD(SUBMODULE_EXPORTS);
   RECORD(SUBMODULE_REQUIRES);
   RECORD(SUBMODULE_EXCLUDED_HEADER);
@@ -2865,6 +2866,14 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
       Stream.EmitRecord(SUBMODULE_IMPORTS, Record);
     }
 
+    // Emit the modules affecting compilation that were not imported.
+    if (!Mod->AffectingModules.empty()) {
+      RecordData Record;
+      for (auto *I : Mod->AffectingModules)
+        Record.push_back(getSubmoduleID(I));
+      Stream.EmitRecord(SUBMODULE_AFFECTING_MODULES, Record);
+    }
+
     // Emit the exports.
     if (!Mod->Exports.empty()) {
       RecordData Record;

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -278,6 +278,11 @@ void ModuleDepCollectorPP::EndOfMainFile() {
   if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
     MDC.addFileDep(MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);
 
+  for (const Module *M :
+       MDC.ScanInstance.getPreprocessor().getAffectingModules())
+    if (!MDC.isPrebuiltModule(M))
+      DirectModularDeps.insert(M);
+
   for (const Module *M : DirectModularDeps) {
     // A top-level module might not be actually imported as a module when
     // -fmodule-name is used to compile a translation unit that imports this
@@ -389,6 +394,8 @@ ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
   llvm::DenseSet<const Module *> AddedModules;
   addAllSubmoduleDeps(M, MD, AddedModules);
+  llvm::DenseSet<const Module *> ProcessedModules;
+  addAllAffectingModules(M, MD, ProcessedModules);
 
   MD.BuildInvocation = MDC.makeInvocationForModuleBuildWithoutOutputs(
       MD, [&](CompilerInvocation &BuildInvocation) {
@@ -461,6 +468,30 @@ void ModuleDepCollectorPP::addModuleDep(
   }
 }
 
+void ModuleDepCollectorPP::addAllAffectingModules(
+    const Module *M, ModuleDeps &MD,
+    llvm::DenseSet<const Module *> &AddedModules) {
+  addAffectingModule(M, MD, AddedModules);
+
+  for (const Module *SubM : M->submodules())
+    addAllAffectingModules(SubM, MD, AddedModules);
+}
+
+void ModuleDepCollectorPP::addAffectingModule(
+    const Module *M, ModuleDeps &MD,
+    llvm::DenseSet<const Module *> &AddedModules) {
+  for (const Module *Affecting : M->AffectingModules) {
+    assert(Affecting == Affecting->getTopLevelModule() &&
+           "Not quite import not top-level module");
+    if (Affecting != M->getTopLevelModule() &&
+        !MDC.isPrebuiltModule(Affecting)) {
+      ModuleID ImportID = handleTopLevelModule(Affecting);
+      if (AddedModules.insert(Affecting).second)
+        MD.ClangModuleDeps.push_back(ImportID);
+    }
+  }
+}
+
 ModuleDepCollector::ModuleDepCollector(
     std::unique_ptr<DependencyOutputOptions> Opts,
     CompilerInstance &ScanInstance, DependencyConsumer &C,