Merge pull request #23844 from nathawes/parseable-interface-cherry-picks

[5.1][03-18-2019] Parseable interface cherrypicks from 5.1

Author: Nathan Hawes

include/swift/Frontend/ParseableInterfaceModuleLoader.h

@@ -110,13 +110,18 @@ namespace clang {
   class CompilerInstance;
 }
 
+namespace unittest {
+  class ParseableInterfaceModuleLoaderTest;
+}
+
 namespace swift {
 
 /// A ModuleLoader that runs a subordinate \c CompilerInvocation and
 /// \c CompilerInstance to convert .swiftinterface files to .swiftmodule
 /// files on the fly, caching the resulting .swiftmodules in the module cache
 /// directory, and loading the serialized .swiftmodules from there.
 class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
+  friend class unittest::ParseableInterfaceModuleLoaderTest;
   explicit ParseableInterfaceModuleLoader(ASTContext &ctx, StringRef cacheDir,
                                           StringRef prebuiltCacheDir,
                                           DependencyTracker *tracker,
@@ -134,6 +139,7 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
     std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer) override;
 
+  bool isCached(StringRef DepPath) override;
 
 public:
   static std::unique_ptr<ParseableInterfaceModuleLoader>
@@ -152,7 +158,7 @@ class ParseableInterfaceModuleLoader : public SerializedModuleLoaderBase {
   static bool buildSwiftModuleFromSwiftInterface(
     ASTContext &Ctx, StringRef CacheDir, StringRef PrebuiltCacheDir,
     StringRef ModuleName, StringRef InPath, StringRef OutPath,
-    bool SerializeDependencyHashes);
+    bool SerializeDependencyHashes, bool TrackSystemDependencies);
 };
 
 /// Extract the specified-or-defaulted -module-cache-path that winds up in

include/swift/Serialization/ModuleFormat.h

@@ -52,7 +52,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 477; // Last change: prebuilt module cache
+const uint16_t SWIFTMODULE_VERSION_MINOR = 478; // SDK-relative dependencies flag
 
 using DeclIDField = BCFixed<31>;
 
@@ -674,6 +674,7 @@ namespace input_block {
     FileSizeField,                 // file size (for validation)
     FileModTimeOrContentHashField, // mtime or content hash (for validation)
     BCFixed<1>,                    // are we reading mtime (0) or hash (1)?
+    BCFixed<1>,                    // SDK-relative?
     BCBlob                         // path
   >;
 }

include/swift/Serialization/SerializationOptions.h

@@ -39,11 +39,14 @@ namespace swift {
     /// appropriate strategy for how to verify if it's up-to-date.
     class FileDependency {
       /// The size of the file on disk, in bytes.
-      uint64_t Size : 63;
+      uint64_t Size : 62;
 
       /// A dependency can be either hash-based or modification-time-based.
       bool IsHashBased : 1;
 
+      /// The dependency path can be absolute or relative to the SDK
+      bool IsSDKRelative : 1;
+
       union {
         /// The last modification time of the file.
         uint64_t ModificationTime;
@@ -56,22 +59,22 @@ namespace swift {
       std::string Path;
 
       FileDependency(uint64_t size, bool isHash, uint64_t hashOrModTime,
-                     StringRef path):
-        Size(size), IsHashBased(isHash), ModificationTime(hashOrModTime),
-        Path(path) {}
+                     StringRef path, bool isSDKRelative):
+        Size(size), IsHashBased(isHash), IsSDKRelative(isSDKRelative),
+        ModificationTime(hashOrModTime), Path(path) {}
     public:
       FileDependency() = delete;
 
       /// Creates a new hash-based file dependency.
       static FileDependency
-      hashBased(StringRef path, uint64_t size, uint64_t hash) {
-        return FileDependency(size, /*isHash*/true, hash, path);
+      hashBased(StringRef path, bool isSDKRelative, uint64_t size, uint64_t hash) {
+        return FileDependency(size, /*isHash*/true, hash, path, isSDKRelative);
       }
 
       /// Creates a new modification time-based file dependency.
       static FileDependency
-      modTimeBased(StringRef path, uint64_t size, uint64_t mtime) {
-        return FileDependency(size, /*isHash*/false, mtime, path);
+      modTimeBased(StringRef path, bool isSDKRelative, uint64_t size, uint64_t mtime) {
+        return FileDependency(size, /*isHash*/false, mtime, path, isSDKRelative);
       }
 
       /// Updates the last-modified time of this dependency.
@@ -94,6 +97,9 @@ namespace swift {
       /// based on content hash.
       bool isHashBased() const { return IsHashBased; }
 
+      /// Determines if this dependency is absolute or relative to the SDK.
+      bool isSDKRelative() const { return IsSDKRelative; }
+
       /// Determines if this dependency is hash-based and should be validated
       /// based on modification time.
       bool isModificationTimeBased() const { return !IsHashBased; }

include/swift/Serialization/SerializedModuleLoader.h

@@ -63,6 +63,11 @@ class SerializedModuleLoaderBase : public ModuleLoader {
                   std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer);
 
+  std::error_code
+  openModuleDocFile(AccessPathElem ModuleID,
+                    StringRef ModuleDocPath,
+                    std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer);
+
   /// If the module loader subclass knows that all options have been tried for
   /// loading an architecture-specific file out of a swiftmodule bundle, try
   /// to list the architectures that \e are present.
@@ -75,6 +80,12 @@ class SerializedModuleLoaderBase : public ModuleLoader {
     return false;
   }
 
+  /// Determines if the provided path is a cached artifact for dependency
+  /// tracking purposes.
+  virtual bool isCached(StringRef DepPath) {
+    return false;
+  }
+
 public:
   virtual ~SerializedModuleLoaderBase();
   SerializedModuleLoaderBase(const SerializedModuleLoaderBase &) = delete;

include/swift/Subsystems.h

@@ -264,6 +264,18 @@ namespace swift {
   void serialize(ModuleOrSourceFile DC, const SerializationOptions &options,
                  const SILModule *M = nullptr);
 
+  /// Serializes a module or single source file to the given output file and
+  /// returns back the file's contents as a memory buffer.
+  ///
+  /// Use this if you intend to immediately load the serialized module, as that
+  /// will both avoid extra filesystem traffic and will ensure you read back
+  /// exactly what was written.
+  void serializeToBuffers(ModuleOrSourceFile DC,
+                          const SerializationOptions &opts,
+                          std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
+                          std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
+                          const SILModule *M = nullptr);
+
   /// Get the CPU, subtarget feature options, and triple to use when emitting code.
   std::tuple<llvm::TargetOptions, std::string, std::vector<std::string>,
              std::string>

lib/AST/ModuleLoader.cpp

@@ -34,9 +34,8 @@ void
 DependencyTracker::addDependency(StringRef File, bool IsSystem) {
   // DependencyTracker exposes an interface that (intentionally) does not talk
   // about clang at all, nor about missing deps. It does expose an IsSystem
-  // dimension, though it is presently always false, we accept it and pass it
-  // along to the clang DependencyCollector in case Swift callers start setting
-  // it to true someday.
+  // dimension, which we accept and pass along to the clang DependencyCollector.
+  // along to the clang DependencyCollector.
   clangCollector->maybeAddDependency(File, /*FromModule=*/false,
                                      IsSystem, /*IsModuleFile=*/false,
                                      /*IsMissing=*/false);

lib/ClangImporter/ClangImporter.cpp

@@ -407,6 +407,35 @@ void ClangImporter::clearTypeResolver() {
 
 #pragma mark Module loading
 
+/// Finds the glibc.modulemap file relative to the provided resource dir.
+///
+/// Note that the module map used for Glibc depends on the target we're
+/// compiling for, and is not included in the resource directory with the other
+/// implicit module maps. It's at {freebsd|linux}/{arch}/glibc.modulemap.
+static Optional<StringRef>
+getGlibcModuleMapPath(StringRef resourceDir, llvm::Triple triple,
+                      SmallVectorImpl<char> &scratch) {
+  if (resourceDir.empty())
+    return None;
+
+  scratch.append(resourceDir.begin(), resourceDir.end());
+  llvm::sys::path::append(
+      scratch,
+      swift::getPlatformNameForTriple(triple),
+      swift::getMajorArchitectureName(triple),
+      "glibc.modulemap");
+
+  // Only specify the module map if that file actually exists.
+  // It may not--for example in the case that
+  // `swiftc -target x86_64-unknown-linux-gnu -emit-ir` is invoked using
+  // a Swift compiler not built for Linux targets.
+  if (llvm::sys::fs::exists(scratch)) {
+    return StringRef(scratch.data(), scratch.size());
+  } else {
+    return None;
+  }
+}
+
 static void
 getNormalInvocationArguments(std::vector<std::string> &invocationArgStrs,
                              ASTContext &ctx,
@@ -571,28 +600,10 @@ getNormalInvocationArguments(std::vector<std::string> &invocationArgStrs,
       }
     }
 
-    // The module map used for Glibc depends on the target we're compiling for,
-    // and is not included in the resource directory with the other implicit
-    // module maps. It's at {freebsd|linux}/{arch}/glibc.modulemap.
     SmallString<128> GlibcModuleMapPath;
-    GlibcModuleMapPath = searchPathOpts.RuntimeResourcePath;
-
-    // Running without a resource directory is not a supported configuration.
-    assert(!GlibcModuleMapPath.empty());
-
-    llvm::sys::path::append(
-      GlibcModuleMapPath,
-      swift::getPlatformNameForTriple(triple),
-      swift::getMajorArchitectureName(triple),
-      "glibc.modulemap");
-
-    // Only specify the module map if that file actually exists.
-    // It may not--for example in the case that
-    // `swiftc -target x86_64-unknown-linux-gnu -emit-ir` is invoked using
-    // a Swift compiler not built for Linux targets.
-    if (llvm::sys::fs::exists(GlibcModuleMapPath)) {
-      invocationArgStrs.push_back(
-        (Twine("-fmodule-map-file=") + GlibcModuleMapPath).str());
+    if (auto path = getGlibcModuleMapPath(searchPathOpts.RuntimeResourcePath,
+                                          triple, GlibcModuleMapPath)) {
+      invocationArgStrs.push_back((Twine("-fmodule-map-file=") + *path).str());
     } else {
       // FIXME: Emit a warning of some kind.
     }