Serialize module dependencies in swift module files

include/swift/AST/ASTContext.h

@@ -1074,6 +1074,9 @@ class ASTContext final {
   /// Does any proper bookkeeping to keep all module loaders up to date as well.
   void addSearchPath(StringRef searchPath, bool isFramework, bool isSystem);
 
+  /// Adds the path to the explicitly built module \c name.
+  void addExplicitModulePath(StringRef name, std::string path);
+
   /// Adds a module loader to this AST context.
   ///
   /// \param loader The new module loader, which will be added after any
@@ -1087,7 +1090,7 @@ class ASTContext final {
   ///                interface.
   void addModuleLoader(std::unique_ptr<ModuleLoader> loader,
                        bool isClang = false, bool isDWARF = false,
-                       bool IsInterface = false);
+                       bool IsInterface = false, bool IsExplicit = false);
 
   /// Add a module interface checker to use for this AST context.
   void addModuleInterfaceChecker(std::unique_ptr<ModuleInterfaceChecker> checker);

include/swift/AST/Module.h

@@ -858,6 +858,13 @@ class ModuleDecl
   /// returns true.
   bool isSubmoduleOf(const ModuleDecl *M) const;
 
+private:
+  std::string CacheKey;
+
+public:
+  void setCacheKey(const std::string &key) { CacheKey = key; }
+  StringRef getCacheKey() const { return CacheKey; }
+
   bool isResilient() const {
     return getResilienceStrategy() != ResilienceStrategy::Default;
   }

include/swift/AST/SearchPathOptions.h

@@ -45,7 +45,7 @@ enum class ModuleLoadingMode {
   PreferInterface,
   PreferSerialized,
   OnlyInterface,
-  OnlySerialized
+  OnlySerialized,
 };
 
 /// A single module search path that can come from different sources, e.g.

include/swift/Basic/LangOptions.h

@@ -459,6 +459,9 @@ namespace swift {
     bool EnableDeserializationSafety =
       ::getenv("SWIFT_ENABLE_DESERIALIZATION_SAFETY");
 
+    /// Disable injecting deserializes module paths into the explict module map.
+    bool DisableDeserializationOfExplicitPaths = false;
+
     /// Attempt to recover for imported modules with broken modularization
     /// in an unsafe way. Currently applies only to xrefs where the target
     /// decl moved to a different module that is already loaded.

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -152,8 +152,9 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
                   std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-                  bool isCanImportLookup, bool isTestableDependencyLookup,
-                  bool &isFramework, bool &isSystemModule) override;
+                  std::string *cacheKey, bool isCanImportLookup,
+                  bool isTestableDependencyLookup, bool &isFramework,
+                  bool &isSystemModule) override;
 
   std::error_code findModuleFilesInDirectory(
       ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
@@ -181,6 +182,7 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
          const llvm::StringMap<std::string> &ExplicitSwiftModuleInputs,
          bool IgnoreSwiftSourceInfoFile);
 
+  void addExplicitModulePath(StringRef name, std::string path) override;
   /// Append visible module names to \p names. Note that names are possibly
   /// duplicated, and not guaranteed to be ordered in any way.
   void collectVisibleTopLevelModuleNames(
@@ -201,8 +203,9 @@ class ExplicitCASModuleLoader : public SerializedModuleLoaderBase {
                   std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-                  bool isCanImportLookup, bool isTestableDependencyLookup,
-                  bool &isFramework, bool &isSystemModule) override;
+                  std::string *cacheKey, bool isCanImportLookup,
+                  bool isTestableDependencyLookup, bool &isFramework,
+                  bool &isSystemModule) override;
 
   std::error_code findModuleFilesInDirectory(
       ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,

include/swift/Serialization/SerializedModuleLoader.h

@@ -103,8 +103,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
       std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-      bool isCanImportLookup, bool isTestableDependencyLookup,
-      bool &isFramework, bool &isSystemModule);
+      std::string *CacheKey, bool isCanImportLookup,
+      bool isTestableDependencyLookup, bool &isFramework, bool &isSystemModule);
 
   /// Attempts to search the provided directory for a loadable serialized
   /// .swiftmodule with the provided `ModuleFilename`. Subclasses must
@@ -268,6 +268,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// A textual reason why the compiler rejected a binary module load
   /// attempt with a given status, to be used for diagnostic output.
   static std::optional<std::string> invalidModuleReason(serialization::Status status);
+
+  virtual void addExplicitModulePath(StringRef name, std::string path) {};
 };
 
 /// Imports serialized Swift modules into an ASTContext.

lib/AST/ASTContext.cpp

@@ -19,8 +19,8 @@
 #include "ClangTypeConverter.h"
 #include "ForeignRepresentationInfo.h"
 #include "SubstitutionMapStorage.h"
-#include "swift/AST/ASTContextGlobalCache.h"
 #include "swift/ABI/MetadataValues.h"
+#include "swift/AST/ASTContextGlobalCache.h"
 #include "swift/AST/AvailabilityContextStorage.h"
 #include "swift/AST/ClangModuleLoader.h"
 #include "swift/AST/ConcreteDeclRef.h"
@@ -67,6 +67,8 @@
 #include "swift/Basic/Statistic.h"
 #include "swift/Basic/StringExtras.h"
 #include "swift/ClangImporter/ClangModule.h"
+#include "swift/Frontend/ModuleInterfaceLoader.h"
+#include "swift/Serialization/SerializedModuleLoader.h"
 #include "swift/Strings.h"
 #include "swift/Subsystems.h"
 #include "swift/SymbolGraphGen/SymbolGraphOptions.h"
@@ -445,6 +447,9 @@ struct ASTContext::Implementation {
   /// Singleton used to cache the import graph.
   swift::namelookup::ImportCache TheImportCache;
 
+  /// The module loader used to load explicit Swift modules.
+  SerializedModuleLoaderBase *TheExplicitSwiftModuleLoader = nullptr;
+
   /// The module loader used to load Clang modules.
   ClangModuleLoader *TheClangModuleLoader = nullptr;
 
@@ -2159,14 +2164,23 @@ void ASTContext::addSearchPath(StringRef searchPath, bool isFramework,
     clangLoader->addSearchPath(searchPath, isFramework, isSystem);
 }
 
+void ASTContext::addExplicitModulePath(StringRef name, std::string path) {
+  if (getImpl().TheExplicitSwiftModuleLoader)
+    getImpl().TheExplicitSwiftModuleLoader->addExplicitModulePath(name, path);
+}
+
 void ASTContext::addModuleLoader(std::unique_ptr<ModuleLoader> loader,
-                                 bool IsClang, bool IsDwarf, bool IsInterface) {
+                                 bool IsClang, bool IsDwarf, bool IsInterface,
+                                 bool IsExplicit) {
   if (IsClang && !IsDwarf && !getImpl().TheClangModuleLoader)
     getImpl().TheClangModuleLoader =
         static_cast<ClangModuleLoader *>(loader.get());
   if (IsClang && IsDwarf && !getImpl().TheDWARFModuleLoader)
     getImpl().TheDWARFModuleLoader =
         static_cast<ClangModuleLoader *>(loader.get());
+  if (IsExplicit && !getImpl().TheExplicitSwiftModuleLoader)
+    getImpl().TheExplicitSwiftModuleLoader =
+        static_cast<SerializedModuleLoaderBase *>(loader.get());
   getImpl().ModuleLoaders.push_back(std::move(loader));
 }
 

lib/DependencyScan/ScanDependencies.cpp

@@ -1497,7 +1497,6 @@ bool swift::dependencies::scanDependencies(CompilerInstance &CI) {
       ctx.Allocate<SwiftDependencyScanningService>();
   ModuleDependenciesCache cache(CI.getMainModule()->getNameStr().str(),
                                 CI.getInvocation().getModuleScanningHash());
-
   if (service->setupCachingDependencyScanningService(CI))
     return true;
 

lib/Frontend/Frontend.cpp

@@ -870,7 +870,7 @@ bool CompilerInstance::setUpModuleLoaders() {
   // Install an explicit module loader if it was created earlier.
   if (ESML) {
     this->DefaultSerializedLoader = ESML.get();
-    Context->addModuleLoader(std::move(ESML));
+    Context->addModuleLoader(std::move(ESML), false, false, false, true);
   }
 
   if (!ExplicitModuleBuild) {

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -2304,8 +2304,8 @@ bool ExplicitSwiftModuleLoader::findModule(
     std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-    bool IsCanImportLookup, bool isTestableDependencyLookup,
-    bool &IsFramework, bool &IsSystemModule) {
+    std::string *cacheKey, bool IsCanImportLookup,
+    bool isTestableDependencyLookup, bool &IsFramework, bool &IsSystemModule) {
   // Find a module with an actual, physical name on disk, in case
   // -module-alias is used (otherwise same).
   //
@@ -2473,6 +2473,12 @@ ExplicitSwiftModuleLoader::create(ASTContext &ctx,
   return result;
 }
 
+void ExplicitSwiftModuleLoader::addExplicitModulePath(StringRef name,
+                                                      std::string path) {
+  ExplicitSwiftModuleInputInfo entry(path, {}, {}, {});
+  Impl.ExplicitModuleMap.try_emplace(name, std::move(entry));
+}
+
 struct ExplicitCASModuleLoader::Implementation {
   ASTContext &Ctx;
   llvm::BumpPtrAllocator Allocator;
@@ -2660,8 +2666,8 @@ bool ExplicitCASModuleLoader::findModule(
     std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-    bool IsCanImportLookup, bool IsTestableDependencyLookup,
-    bool &IsFramework, bool &IsSystemModule) {
+    std::string *CacheKey, bool IsCanImportLookup,
+    bool IsTestableDependencyLookup, bool &IsFramework, bool &IsSystemModule) {
   // Find a module with an actual, physical name on disk, in case
   // -module-alias is used (otherwise same).
   //
@@ -2681,6 +2687,8 @@ bool ExplicitCASModuleLoader::findModule(
   // Set IsFramework bit according to the moduleInfo
   IsFramework = moduleInfo.isFramework;
   IsSystemModule = moduleInfo.isSystem;
+  if (CacheKey && moduleInfo.moduleCacheKey)
+    *CacheKey = *moduleInfo.moduleCacheKey;
 
   // Fallback check for module cache key passed on command-line as module path.
   std::string moduleCASID = moduleInfo.moduleCacheKey

lib/Serialization/ModuleFile.cpp

@@ -210,6 +210,10 @@ ModuleFile::loadDependenciesForFileContext(const FileUnit *file,
     auto importPath = builder.copyTo(ctx);
     auto modulePath = importPath.getModulePath(dependency.isScoped());
     auto accessPath = importPath.getAccessPath(dependency.isScoped());
+    if (!getContext().LangOpts.DisableDeserializationOfExplicitPaths &&
+        !dependency.Core.BinaryModulePath.empty())
+      ctx.addExplicitModulePath(modulePath.front().Item.str(),
+                                dependency.Core.BinaryModulePath.str());
 
     auto module = getModule(modulePath, /*allowLoading*/true);
     if (!module || module->failedToLoad()) {

lib/Serialization/ModuleFileSharedCore.cpp

@@ -1561,9 +1561,9 @@ ModuleFileSharedCore::ModuleFileSharedCore(
           unsigned rawImportControl;
           bool scoped;
           bool hasSPI;
-          input_block::ImportedModuleLayout::readRecord(scratch,
-                                                        rawImportControl,
-                                                        scoped, hasSPI);
+          bool hasPath;
+          input_block::ImportedModuleLayout::readRecord(
+              scratch, rawImportControl, scoped, hasSPI, hasPath);
           auto importKind = getActualImportControl(rawImportControl);
           if (!importKind) {
             // We don't know how to import this dependency.
@@ -1580,14 +1580,25 @@ ModuleFileSharedCore::ModuleFileSharedCore(
             unsigned recordID = fatalIfUnexpected(
                 cursor.readRecord(entry.ID, scratch, &spiBlob));
             assert(recordID == input_block::IMPORTED_MODULE_SPIS);
-            input_block::ImportedModuleLayoutSPI::readRecord(scratch);
-            (void) recordID;
-          } else {
-            spiBlob = StringRef();
+            input_block::ImportedModuleSPILayout::readRecord(scratch);
+            (void)recordID;
+          }
+
+          StringRef pathBlob;
+          if (hasPath) {
+            scratch.clear();
+
+            llvm::BitstreamEntry entry =
+                fatalIfUnexpected(cursor.advance(AF_DontPopBlockAtEnd));
+            unsigned recordID = fatalIfUnexpected(
+                cursor.readRecord(entry.ID, scratch, &pathBlob));
+            assert(recordID == input_block::IMPORTED_MODULE_PATH);
+            input_block::ImportedModulePathLayout::readRecord(scratch);
+            (void)recordID;
           }
 
           Dependencies.push_back(
-              {blobData, spiBlob, importKind.value(), scoped});
+              {blobData, spiBlob, pathBlob, importKind.value(), scoped});
           break;
         }
         case input_block::LINK_LIBRARY: {

lib/Serialization/ModuleFileSharedCore.h

@@ -123,6 +123,7 @@ class ModuleFileSharedCore {
   public:
     const StringRef RawPath;
     const StringRef RawSPIs;
+    const StringRef BinaryModulePath;
 
   private:
     using ImportFilterKind = ModuleDecl::ImportFilterKind;
@@ -137,27 +138,26 @@ class ModuleFileSharedCore {
       return static_cast<ImportFilterKind>(1 << RawImportControl);
     }
 
-    Dependency(StringRef path, StringRef spiGroups, bool isHeader,
-               ImportFilterKind importControl, bool isScoped)
-        : RawPath(path),
-          RawSPIs(spiGroups),
+    Dependency(StringRef path, StringRef spiGroups, StringRef binaryModulePath,
+               bool isHeader, ImportFilterKind importControl, bool isScoped)
+        : RawPath(path), RawSPIs(spiGroups), BinaryModulePath(binaryModulePath),
           RawImportControl(rawControlFromKind(importControl)),
-          IsHeader(isHeader),
-          IsScoped(isScoped) {
+          IsHeader(isHeader), IsScoped(isScoped) {
       assert(llvm::popcount(static_cast<unsigned>(importControl)) == 1 &&
              "must be a particular filter option, not a bitset");
       assert(getImportControl() == importControl && "not enough bits");
     }
 
   public:
-   Dependency(StringRef path, StringRef spiGroups,
-              ImportFilterKind importControl, bool isScoped)
-       : Dependency(path, spiGroups, false, importControl, isScoped) {}
-
-   static Dependency forHeader(StringRef headerPath, bool exported) {
-     auto importControl =
-         exported ? ImportFilterKind::Exported : ImportFilterKind::Default;
-     return Dependency(headerPath, StringRef(), true, importControl, false);
+    Dependency(StringRef path, StringRef spiGroups, StringRef binaryModulePath,
+               ImportFilterKind importControl, bool isScoped)
+        : Dependency(path, spiGroups, binaryModulePath, false, importControl,
+                     isScoped) {}
+
+    static Dependency forHeader(StringRef headerPath, bool exported) {
+      auto importControl =
+          exported ? ImportFilterKind::Exported : ImportFilterKind::Default;
+      return Dependency(headerPath, {}, {}, true, importControl, false);
     }
 
     bool isExported() const {

lib/Serialization/ModuleFormat.h

@@ -58,7 +58,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 968; // @inout result convention
+const uint16_t SWIFTMODULE_VERSION_MINOR = 969; // Module import paths
 
 /// A standard hash seed used for all string hashes in a serialized module.
 ///
@@ -1131,6 +1131,7 @@ namespace input_block {
     DEPENDENCY_DIRECTORY,
     MODULE_INTERFACE_PATH,
     IMPORTED_MODULE_SPIS,
+    IMPORTED_MODULE_PATH,
     EXTERNAL_MACRO,
   };
 
@@ -1139,16 +1140,22 @@ namespace input_block {
     ImportControlField, // import kind
     BCFixed<1>,         // scoped?
     BCFixed<1>,         // has spis?
-    BCBlob // module name, with submodule path pieces separated by \0s.
-           // If the 'scoped' flag is set, the final path piece is an access
-           // path within the module.
+    BCFixed<1>,         // has path?
+    BCBlob //  module name, with submodule path pieces separated by \0s.  If the
+           // 'scoped' flag is set, the final path piece is an access path
+           // within the module.
   >;
 
-  using ImportedModuleLayoutSPI = BCRecordLayout<
+  using ImportedModuleSPILayout = BCRecordLayout<
     IMPORTED_MODULE_SPIS,
     BCBlob // SPI names, separated by \0s
   >;
 
+  using ImportedModulePathLayout = BCRecordLayout<
+    IMPORTED_MODULE_PATH,
+    BCBlob // Module file path
+  >;
+
   using ExternalMacroLayout = BCRecordLayout<
     EXTERNAL_MACRO,
     ImportControlField, // import kind