Support Swift CompileJob Caching

Sources/CSwiftScan/include/swiftscan_header.h

@@ -18,7 +18,7 @@
 #include <stdint.h>
 
 #define SWIFTSCAN_VERSION_MAJOR 0
-#define SWIFTSCAN_VERSION_MINOR 1
+#define SWIFTSCAN_VERSION_MINOR 4
 
 //=== Public Scanner Data Types -------------------------------------------===//
 
@@ -77,6 +77,18 @@ typedef struct {
 typedef struct swiftscan_scan_invocation_s *swiftscan_scan_invocation_t;
 typedef void *swiftscan_scanner_t;
 
+//=== CAS/Caching Specification -------------------------------------------===//
+typedef struct swiftscan_cas_s *swiftscan_cas_t;
+
+typedef enum {
+  SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3,
+  SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4,
+  SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5
+} swiftscan_output_kind_t;
+
 //=== libSwiftScan Functions ------------------------------------------------===//
 
 typedef struct {
@@ -117,13 +129,17 @@ typedef struct {
   swiftscan_string_set_t *
   (*swiftscan_swift_textual_detail_get_command_line)(swiftscan_module_details_t);
   swiftscan_string_set_t *
+  (*swiftscan_swift_textual_detail_get_bridging_pch_command_line)(swiftscan_module_details_t);
+  swiftscan_string_set_t *
   (*swiftscan_swift_textual_detail_get_extra_pcm_args)(swiftscan_module_details_t);
   swiftscan_string_ref_t
   (*swiftscan_swift_textual_detail_get_context_hash)(swiftscan_module_details_t);
   bool
   (*swiftscan_swift_textual_detail_get_is_framework)(swiftscan_module_details_t);
   swiftscan_string_set_t *
   (*swiftscan_swift_textual_detail_get_swift_overlay_dependencies)(swiftscan_module_details_t);
+  swiftscan_string_ref_t
+  (*swiftscan_swift_textual_detail_get_module_cache_key)(swiftscan_module_details_t);
 
   //=== Swift Binary Module Details query APIs ------------------------------===//
   swiftscan_string_ref_t
@@ -136,6 +152,8 @@ typedef struct {
   (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t);
   bool
   (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t);
+  swiftscan_string_ref_t
+  (*swiftscan_swift_binary_detail_get_module_cache_key)(swiftscan_module_details_t);
 
   //=== Swift Placeholder Module Details query APIs -------------------------===//
   swiftscan_string_ref_t
@@ -154,6 +172,8 @@ typedef struct {
   (*swiftscan_clang_detail_get_command_line)(swiftscan_module_details_t);
   swiftscan_string_set_t *
   (*swiftscan_clang_detail_get_captured_pcm_args)(swiftscan_module_details_t);
+  swiftscan_string_ref_t
+  (*swiftscan_clang_detail_get_module_cache_key)(swiftscan_module_details_t);
 
   //=== Batch Scan Input Functions ------------------------------------------===//
   swiftscan_batch_scan_input_t *
@@ -253,6 +273,17 @@ typedef struct {
   bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
   void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
 
+  //=== Scanner CAS Operations ----------------------------------------------===//
+  swiftscan_cas_t (*swiftscan_cas_create)(const char *path);
+  void (*swiftscan_cas_dispose)(swiftscan_cas_t cas);
+  swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas,
+                                                uint8_t *data, unsigned size);
+  swiftscan_string_ref_t (*swiftscan_compute_cache_key)(swiftscan_cas_t cas,
+                                                        int argc,
+                                                        const char *argv,
+                                                        const char *input,
+                                                        swiftscan_output_kind_t);
+
 } swiftscan_functions_t;
 
 #endif // SWIFT_C_DEPENDENCY_SCAN_H

Sources/SwiftDriver/Driver/Driver.swift

@@ -74,6 +74,8 @@ public struct Driver {
     case missingContextHashOnSwiftDependency(String)
     case dependencyScanningFailure(Int, String)
     case missingExternalDependency(String)
+    // Compiler Caching Failures
+    case unsupportedConfigurationForCaching(String)
 
     public var description: String {
       switch self {
@@ -135,6 +137,8 @@ public struct Driver {
         return "unable to load output file map '\(path)': \(error)"
       case .missingExternalDependency(let moduleName):
         return "Missing External dependency info for module: \(moduleName)"
+      case .unsupportedConfigurationForCaching(let reason):
+        return "unsupported configuration for -cache-compile-job: \(reason)"
       case .baselineGenerationRequiresTopLevelModule(let arg):
         return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'"
       case .optionRequiresAnother(let first, let second):
@@ -263,6 +267,11 @@ public struct Driver {
   /// Whether to consider incremental compilation.
   let shouldAttemptIncrementalCompilation: Bool
 
+  /// CAS/Caching related options.
+  let enableCaching: Bool
+  let useClangIncludeTree: Bool
+  let casPath: String
+
   /// Code & data for incremental compilation. Nil if not running in incremental mode.
   /// Set during planning because needs the jobs to look at outputs.
   @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil
@@ -571,6 +580,17 @@ public struct Driver {
                                                                                         diagnosticEngine: diagnosticsEngine,
                                                                                         compilerMode: compilerMode)
 
+    let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
+    self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
+    self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
+    if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle {
+      self.casPath = casPathOpt.description
+    } else if let cacheEnv = env["CCHROOT"] {
+      self.casPath = cacheEnv
+    } else {
+      self.casPath = ""
+    }
+
     // Compute the working directory.
     workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
       let cwd = fileSystem.currentWorkingDirectory

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -45,6 +45,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
   /// Whether we are using the integrated driver via libSwiftDriver shared lib
   private let integratedDriver: Bool
   private let mainModuleName: String?
+  private let enableCAS: Bool
+  private let swiftScanOracle: InterModuleDependencyOracle
 
   /// Clang PCM names contain a hash of the command-line arguments that were used to build them.
   /// We avoid re-running the hash computation with the use of this cache
@@ -55,14 +57,23 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
 
   public init(dependencyGraph: InterModuleDependencyGraph,
               toolchain: Toolchain,
+              dependencyOracle: InterModuleDependencyOracle,
               integratedDriver: Bool = true,
-              supportsExplicitInterfaceBuild: Bool = false) throws {
+              supportsExplicitInterfaceBuild: Bool = false,
+              enableCAS: Bool = false) throws {
     self.dependencyGraph = dependencyGraph
     self.toolchain = toolchain
+    self.swiftScanOracle = dependencyOracle
     self.integratedDriver = integratedDriver
     self.mainModuleName = dependencyGraph.mainModuleName
     self.reachabilityMap = try dependencyGraph.computeTransitiveClosure()
     self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild
+    self.enableCAS = enableCAS
+  }
+
+  /// Supports resolving bridging header pch command from swiftScan.
+  public func supportsBridgingHeaderPCHCommand() throws -> Bool {
+    return try swiftScanOracle.supportsBridgingHeaderPCHCommand()
   }
 
   /// Generate build jobs for all dependencies of the main module.
@@ -235,7 +246,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
       inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path,
                                      type: .swiftModule))
 
-      for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] {
+      let prebuiltHeaderDependencyPaths = dependencyModule.prebuiltHeaderDependencyPaths ?? []
+      if enableCAS && !prebuiltHeaderDependencyPaths.isEmpty {
+        throw Driver.Error.unsupportedConfigurationForCaching("module \(dependencyModule.moduleName) has prebuilt header dependency")
+      }
+
+      for headerDep in prebuiltHeaderDependencyPaths {
         commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"])
         commandLine.appendPath(VirtualPath.lookup(headerDep.path))
         inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch))
@@ -256,11 +272,21 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
 
     // Swift Main Module dependencies are passed encoded in a JSON file as described by
     // SwiftModuleArtifactInfo
-    if moduleId.moduleName == mainModuleName {
+    guard moduleId == .swift(dependencyGraph.mainModuleName) else { return }
+    let dependencyFileContent =
+      try serializeModuleDependencies(for: moduleId,
+                                      swiftDependencyArtifacts: swiftDependencyArtifacts,
+                                      clangDependencyArtifacts: clangDependencyArtifacts)
+    if enableCAS {
+      // When using a CAS, write JSON into CAS and pass the ID on command-line.
+      let casID = try swiftScanOracle.store(data: dependencyFileContent)
+      commandLine.appendFlag("-explicit-swift-module-map-file")
+      commandLine.appendFlag(casID)
+    } else {
+      // Write JSON to a file and add the JSON artifacts to command-line and inputs.
       let dependencyFile =
-        try serializeModuleDependencies(for: moduleId,
-                                        swiftDependencyArtifacts: swiftDependencyArtifacts,
-                                        clangDependencyArtifacts: clangDependencyArtifacts)
+        VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"),
+                                                               dependencyFileContent)
       commandLine.appendFlag("-explicit-swift-module-map-file")
       commandLine.appendPath(dependencyFile)
       inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
@@ -280,13 +306,15 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
         let isFramework: Bool
         swiftModulePath = .init(file: dependencyInfo.modulePath.path,
                                 type: .swiftModule)
-        isFramework = try dependencyGraph.swiftModuleDetails(of: dependencyId).isFramework ?? false
+        let swiftModuleDetails = try dependencyGraph.swiftModuleDetails(of: dependencyId)
+        isFramework = swiftModuleDetails.isFramework ?? false
         // Accumulate the required information about this dependency
         // TODO: add .swiftdoc and .swiftsourceinfo for this module.
         swiftDependencyArtifacts.append(
           SwiftModuleArtifactInfo(name: dependencyId.moduleName,
                                   modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
-                                  isFramework: isFramework))
+                                  isFramework: isFramework,
+                                  moduleCacheKey: swiftModuleDetails.moduleCacheKey))
       case .clang:
         let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId)
         let dependencyClangModuleDetails =
@@ -295,7 +323,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
         clangDependencyArtifacts.append(
           ClangModuleArtifactInfo(name: dependencyId.moduleName,
                                   modulePath: TextualVirtualPath(path: dependencyInfo.modulePath.path),
-                                  moduleMapPath: dependencyClangModuleDetails.moduleMapPath))
+                                  moduleMapPath: dependencyClangModuleDetails.moduleMapPath,
+                                  moduleCacheKey: dependencyClangModuleDetails.moduleCacheKey))
       case .swiftPrebuiltExternal:
         let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId)
         let compiledModulePath = prebuiltModuleDetails.compiledModulePath
@@ -308,7 +337,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
           SwiftModuleArtifactInfo(name: dependencyId.moduleName,
                                   modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
                                   headerDependencies: prebuiltModuleDetails.headerDependencyPaths,
-                                  isFramework: isFramework))
+                                  isFramework: isFramework,
+                                  moduleCacheKey: prebuiltModuleDetails.moduleCacheKey))
       case .swiftPlaceholder:
         fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")
     }
@@ -354,6 +384,11 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
   public mutating func resolveMainModuleDependencies(inputs: inout [TypedVirtualPath],
                                                      commandLine: inout [Job.ArgTemplate]) throws {
     let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName)
+
+    let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId)
+    if let additionalArgs = mainModuleDetails.commandLine {
+      additionalArgs.forEach { commandLine.appendFlag($0) }
+    }
     commandLine.appendFlags("-disable-implicit-swift-modules",
                             "-Xcc", "-fno-implicit-modules",
                             "-Xcc", "-fno-implicit-module-maps")
@@ -367,11 +402,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
   public mutating func resolveBridgingHeaderDependencies(inputs: inout [TypedVirtualPath],
                                                          commandLine: inout [Job.ArgTemplate]) throws {
     let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName)
-    // Prohibit the frontend from implicitly building textual modules into binary modules.
-    commandLine.appendFlags("-disable-implicit-swift-modules",
-                            "-Xcc", "-fno-implicit-modules",
-                            "-Xcc", "-fno-implicit-module-maps")
-
     var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
     var clangDependencyArtifacts: [ClangModuleArtifactInfo] = []
     let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId)
@@ -409,28 +439,46 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
       inputs.append(clangModuleMapPath)
     }
 
+    // Return if depscanner provided build commands.
+    if let scannerPCHArgs = mainModuleDetails.bridgingPchCommandLine {
+      scannerPCHArgs.forEach { commandLine.appendFlag($0) }
+      return
+    }
+
+    assert(!enableCAS, "Caching build should always return command-line from scanner")
+    // Prohibit the frontend from implicitly building textual modules into binary modules.
+    commandLine.appendFlags("-disable-implicit-swift-modules",
+                            "-Xcc", "-fno-implicit-modules",
+                            "-Xcc", "-fno-implicit-module-maps")
+
+    let dependencyFileContent =
+      try serializeModuleDependencies(for: mainModuleId,
+                                      swiftDependencyArtifacts: swiftDependencyArtifacts,
+                                      clangDependencyArtifacts: clangDependencyArtifacts)
+
     let dependencyFile =
-    try serializeModuleDependencies(for: mainModuleId,
-                                    swiftDependencyArtifacts: swiftDependencyArtifacts,
-                                    clangDependencyArtifacts: clangDependencyArtifacts)
+      VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(mainModuleId.moduleName)-dependencies.json"),
+                                                             dependencyFileContent)
     commandLine.appendFlag("-explicit-swift-module-map-file")
     commandLine.appendPath(dependencyFile)
     inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
                                    type: .jsonSwiftArtifacts))
   }
 
-  /// Store the output file artifacts for a given module in a JSON file, return the file's path.
+  /// Serialize the output file artifacts for a given module in JSON format.
   private func serializeModuleDependencies(for moduleId: ModuleDependencyId,
                                            swiftDependencyArtifacts: [SwiftModuleArtifactInfo],
                                            clangDependencyArtifacts: [ClangModuleArtifactInfo]
-  ) throws -> VirtualPath {
+  ) throws -> Data {
+    // The module dependency map in CAS needs to be stable.
+    // Sort the dependencies by name.
     let allDependencyArtifacts: [ModuleDependencyArtifactInfo] =
-      swiftDependencyArtifacts.map {ModuleDependencyArtifactInfo.swift($0)} +
-      clangDependencyArtifacts.map {ModuleDependencyArtifactInfo.clang($0)}
+      swiftDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.swift($0)} +
+      clangDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.clang($0)}
     let encoder = JSONEncoder()
-    encoder.outputFormatting = [.prettyPrinted]
-    let contents = try encoder.encode(allDependencyArtifacts)
-    return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), contents)
+    // Use sorted key to ensure the order of the keys is stable.
+    encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+    return try encoder.encode(allDependencyArtifacts)
   }
 
   private func getPCMHashParts(pcmArgs: [String], contextHash: String) -> [String] {