Issue Carthage#2400 Shared cache for built dependencies

Source/CarthageKit/Constants.swift

@@ -24,7 +24,7 @@ public struct Constants {
 	}()
 
 	/// ~/Library/Caches/org.carthage.CarthageKit/
-	private static let userCachesURL: URL = {
+	internal static let userCachesURL: URL = {
 		let fileManager = FileManager.default
 
 		let urlResult: Result<URL, NSError> = Result(attempt: {

Source/CarthageKit/Project.swift

@@ -42,6 +42,9 @@ public enum ProjectEvent {
 	/// Building the project is being skipped because it is cached.
 	case skippedBuildingCached(Dependency)
 
+	/// Copying from shared cache
+	case skippedBuildingCopyFromSharedCache(Dependency)
+
 	/// Rebuilding a cached project because of a version file/framework mismatch.
 	case rebuildingCached(Dependency)
 
@@ -128,8 +131,7 @@ public final class Project { // swiftlint:disable:this type_body_length
 	private var cachedBinaryProjects: CachedBinaryProjects = [:]
 	private let cachedBinaryProjectsQueue = SerialProducerQueue(name: "org.carthage.Constants.Project.cachedBinaryProjectsQueue")
 
-	private lazy var xcodeVersionDirectory: String = XcodeVersion.make()
-		.map { "\($0.version)_\($0.buildVersion)" } ?? "Unknown"
+	private lazy var xcodeVersionDirectory: String = XcodeVersion.makeString()
 
 	/// Attempts to load Cartfile or Cartfile.private from the given directory,
 	/// merging their dependencies.
@@ -586,27 +588,7 @@ public final class Project { // swiftlint:disable:this type_body_length
 		return SignalProducer<URL, CarthageError>(value: zipFile)
 			.flatMap(.concat, unarchive(archive:))
 			.flatMap(.concat) { directoryURL -> SignalProducer<URL, CarthageError> in
-				return frameworksInDirectory(directoryURL)
-					.flatMap(.merge) { url -> SignalProducer<URL, CarthageError> in
-						return checkFrameworkCompatibility(url, usingToolchain: toolchain)
-							.mapError { error in CarthageError.internalError(description: error.description) }
-					}
-					.flatMap(.merge, self.copyFrameworkToBuildFolder)
-					.flatMap(.merge) { frameworkURL -> SignalProducer<URL, CarthageError> in
-						return self.copyDSYMToBuildFolderForFramework(frameworkURL, fromDirectoryURL: directoryURL)
-							.then(self.copyBCSymbolMapsToBuildFolderForFramework(frameworkURL, fromDirectoryURL: directoryURL))
-							.then(SignalProducer(value: frameworkURL))
-					}
-					.collect()
-					.flatMap(.concat) { frameworkURLs -> SignalProducer<(), CarthageError> in
-						return self.createVersionFilesForFrameworks(
-							frameworkURLs,
-							fromDirectoryURL: directoryURL,
-							projectName: projectName,
-							commitish: pinnedVersion.commitish
-						)
-					}
-					.then(SignalProducer<URL, CarthageError>(value: directoryURL))
+				return copyFrameworks(in: directoryURL, to: self.directoryURL, projectName: projectName, pinnedVersion: pinnedVersion, toolchain: toolchain)
 			}
 	}
 
@@ -711,55 +693,6 @@ public final class Project { // swiftlint:disable:this type_body_length
 			}
 	}
 
-	/// Copies the framework at the given URL into the current project's build
-	/// folder.
-	///
-	/// Sends the URL to the framework after copying.
-	private func copyFrameworkToBuildFolder(_ frameworkURL: URL) -> SignalProducer<URL, CarthageError> {
-		return platformForFramework(frameworkURL)
-			.flatMap(.merge) { platform -> SignalProducer<URL, CarthageError> in
-				let platformFolderURL = self.directoryURL.appendingPathComponent(platform.relativePath, isDirectory: true)
-				return SignalProducer(value: frameworkURL)
-					.copyFileURLsIntoDirectory(platformFolderURL)
-			}
-	}
-
-	/// Copies the DSYM matching the given framework and contained within the
-	/// given directory URL to the directory that the framework resides within.
-	///
-	/// If no dSYM is found for the given framework, completes with no values.
-	///
-	/// Sends the URL of the dSYM after copying.
-	public func copyDSYMToBuildFolderForFramework(_ frameworkURL: URL, fromDirectoryURL directoryURL: URL) -> SignalProducer<URL, CarthageError> {
-		let destinationDirectoryURL = frameworkURL.deletingLastPathComponent()
-		return dSYMForFramework(frameworkURL, inDirectoryURL: directoryURL)
-			.copyFileURLsIntoDirectory(destinationDirectoryURL)
-	}
-
-	/// Copies any *.bcsymbolmap files matching the given framework and contained
-	/// within the given directory URL to the directory that the framework
-	/// resides within.
-	///
-	/// If no bcsymbolmap files are found for the given framework, completes with
-	/// no values.
-	///
-	/// Sends the URLs of the bcsymbolmap files after copying.
-	public func copyBCSymbolMapsToBuildFolderForFramework(_ frameworkURL: URL, fromDirectoryURL directoryURL: URL) -> SignalProducer<URL, CarthageError> {
-		let destinationDirectoryURL = frameworkURL.deletingLastPathComponent()
-		return BCSymbolMapsForFramework(frameworkURL, inDirectoryURL: directoryURL)
-			.copyFileURLsIntoDirectory(destinationDirectoryURL)
-	}
-
-	/// Creates a .version file for all of the provided frameworks.
-	public func createVersionFilesForFrameworks(
-		_ frameworkURLs: [URL],
-		fromDirectoryURL directoryURL: URL,
-		projectName: String,
-		commitish: String
-	) -> SignalProducer<(), CarthageError> {
-		return createVersionFileForCommitish(commitish, dependencyName: projectName, buildProducts: frameworkURLs, rootDirectoryURL: self.directoryURL)
-	}
-
 	private let gitOperationQueue = SerialProducerQueue(name: "org.carthage.Constants.Project.gitOperationQueue")
 
 	/// Checks out the given dependency into its intended working directory,
@@ -1060,6 +993,21 @@ public final class Project { // swiftlint:disable:this type_body_length
 					return dependenciesIncludingNext
 				}
 			}
+			.flatMap(.concat) { (dependencies: [(Dependency, PinnedVersion)]) -> SignalProducer<[(Dependency, PinnedVersion)], CarthageError> in
+				return self.copyBuiltFrameworksFromSharedCacheIfVersionMatches(for: dependencies, options: options)
+					.flatMap(.concat) { dependency, version -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
+						self._projectEventsObserver.send(value: .skippedBuildingCopyFromSharedCache(dependency))
+						return SignalProducer(value: (dependency, version))
+					}
+					.collect()
+					.map { copiedFromSharedCacheDependencies -> [(Dependency, PinnedVersion)] in
+						// Filters out dependencies that we've copied from shared cache
+						// but preserves the build order
+						return dependencies.filter { dependency -> Bool in
+							!copiedFromSharedCacheDependencies.contains { $0 == dependency }
+						}
+					}
+			}
 			.flatMap(.concat) { (dependencies: [(Dependency, PinnedVersion)]) -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
 				return SignalProducer(dependencies)
 					.flatMap(.concurrent(limit: 4)) { dependency, version -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
@@ -1135,6 +1083,37 @@ public final class Project { // swiftlint:disable:this type_body_length
 			}
 	}
 
+	private func copyBuiltFrameworksFromSharedCacheIfVersionMatches(
+		for dependencies: [(Dependency, PinnedVersion)],
+		options: BuildOptions
+	) -> SignalProducer<(Dependency, PinnedVersion), CarthageError> {
+		return SignalProducer(dependencies)
+			.flatMap(.merge) { dependency, version -> SignalProducer<(Dependency, PinnedVersion, URL, Bool?), CarthageError> in
+				let sharedCacheURLForDependency = sharedCacheURLByXcodeVersionFor(dependency: dependency, pinnedVersion: version)
+
+				return SignalProducer.combineLatest(
+					SignalProducer(value: dependency),
+					SignalProducer(value: version),
+					SignalProducer(value: sharedCacheURLForDependency),
+					versionFileMatches(dependency, version: version, platforms: options.platforms,
+									   rootDirectoryURL: sharedCacheURLForDependency, toolchain: options.toolchain))
+			}
+			.flatMap(.merge) { dependency, version, sharedCacheVersioned, sharedCacheMatches -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
+				if let sharedCacheVersionFileMatches = sharedCacheMatches, sharedCacheVersionFileMatches == true {
+					return copyFrameworks(in: sharedCacheVersioned, to: self.directoryURL, projectName: dependency.name, pinnedVersion: version, toolchain: options.toolchain)
+						.collect()
+						.flatMap(.merge) { urls -> SignalProducer<(Dependency, PinnedVersion), CarthageError> in
+							if urls.isEmpty {
+								return .empty
+							} else {
+								return SignalProducer(value: (dependency, version))
+							}
+					}
+				}
+				return .empty
+			}
+	}
+
 	private func symlinkBuildPathIfNeeded(for dependency: Dependency, version: PinnedVersion) -> SignalProducer<(), CarthageError> {
 		return dependencySet(for: dependency, version: version)
 			.flatMap(.merge) { dependencies -> SignalProducer<(), CarthageError> in
@@ -1374,6 +1353,104 @@ internal func frameworksInDirectory(_ directoryURL: URL) -> SignalProducer<URL,
 	}
 }
 
+internal func copyFrameworksAndCreateVersionFile(
+	_ frameworkURLs: [URL],
+	from directoryURL: URL,
+	to destinationURL: URL,
+	projectName: String,
+	pinnedVersion: PinnedVersion,
+	toolchain: String?
+) -> SignalProducer<URL, CarthageError> {
+	return SignalProducer(value: frameworkURLs)
+		.flatten()
+		.flatMap(.merge) { frameworkURL -> SignalProducer<URL, CarthageError> in
+			copyFramework(frameworkURL, toFolder: destinationURL)
+		}
+		.flatMap(.merge) { frameworkURL -> SignalProducer<URL, CarthageError> in
+			return copyDSYMToBuildFolderForFramework(frameworkURL, fromDirectoryURL: directoryURL)
+				.then(copyBCSymbolMapsToBuildFolderForFramework(frameworkURL, fromDirectoryURL: directoryURL))
+				.then(SignalProducer(value: frameworkURL))
+		}
+		.collect()
+		.flatMap(.concat) { frameworkURLs -> SignalProducer<(), CarthageError> in
+			return createVersionFilesForFrameworks(
+				frameworkURLs,
+				fromDirectoryURL: destinationURL,
+				projectName: projectName,
+				commitish: pinnedVersion.commitish
+			)
+		}
+		.then(SignalProducer<URL, CarthageError>(value: directoryURL))
+}
+
+internal func copyFrameworks(
+	in directoryURL: URL,
+	to destinationURL: URL,
+	projectName: String,
+	pinnedVersion: PinnedVersion,
+	toolchain: String?
+) -> SignalProducer<URL, CarthageError> {
+	return frameworksInDirectory(directoryURL)
+		.flatMap(.merge) { url -> SignalProducer<URL, CarthageError> in
+			return checkFrameworkCompatibility(url, usingToolchain: toolchain)
+				.mapError { error in CarthageError.internalError(description: error.description) }
+		}
+		.collect()
+		.flatMap(.merge, { frameworkURLs -> SignalProducer<URL, CarthageError> in
+			copyFrameworksAndCreateVersionFile(frameworkURLs, from: directoryURL, to: destinationURL,
+											   projectName: projectName, pinnedVersion: pinnedVersion, toolchain: toolchain)
+		})
+}
+
+/// Copies any *.bcsymbolmap files matching the given framework and contained
+/// within the given directory URL to the directory that the framework
+/// resides within.
+///
+/// If no bcsymbolmap files are found for the given framework, completes with
+/// no values.
+///
+/// Sends the URLs of the bcsymbolmap files after copying.
+internal func copyBCSymbolMapsToBuildFolderForFramework(_ frameworkURL: URL, fromDirectoryURL directoryURL: URL) -> SignalProducer<URL, CarthageError> {
+	let destinationDirectoryURL = frameworkURL.deletingLastPathComponent()
+	return BCSymbolMapsForFramework(frameworkURL, inDirectoryURL: directoryURL)
+		.copyFileURLsIntoDirectory(destinationDirectoryURL)
+}
+
+/// Creates a .version file for all of the provided frameworks.
+internal func createVersionFilesForFrameworks(
+	_ frameworkURLs: [URL],
+	fromDirectoryURL directoryURL: URL,
+	projectName: String,
+	commitish: String
+	) -> SignalProducer<(), CarthageError> {
+	return createVersionFileForCommitish(commitish, dependencyName: projectName, buildProducts: frameworkURLs,
+										 rootDirectoryURL: directoryURL)
+}
+
+/// Copies the framework at the given URL into the specified folder.
+///
+/// Sends the URL to the framework after copying.
+internal func copyFramework(_ frameworkURL: URL, toFolder folderURL: URL) -> SignalProducer<URL, CarthageError> {
+	return platformForFramework(frameworkURL)
+		.flatMap(.merge) { platform -> SignalProducer<URL, CarthageError> in
+			let platformFolderURL = folderURL.appendingPathComponent(platform.relativePath, isDirectory: true)
+			return SignalProducer(value: frameworkURL)
+				.copyFileURLsIntoDirectory(platformFolderURL)
+	}
+}
+
+/// Copies the DSYM matching the given framework and contained within the
+/// given directory URL to the directory that the framework resides within.
+///
+/// If no dSYM is found for the given framework, completes with no values.
+///
+/// Sends the URL of the dSYM after copying.
+internal func copyDSYMToBuildFolderForFramework(_ frameworkURL: URL, fromDirectoryURL directoryURL: URL) -> SignalProducer<URL, CarthageError> {
+	let destinationDirectoryURL = frameworkURL.deletingLastPathComponent()
+	return dSYMForFramework(frameworkURL, inDirectoryURL: directoryURL)
+		.copyFileURLsIntoDirectory(destinationDirectoryURL)
+}
+
 /// Sends the URL to each dSYM found in the given directory
 internal func dSYMsInDirectory(_ directoryURL: URL) -> SignalProducer<URL, CarthageError> {
 	return filesInDirectory(directoryURL, "com.apple.xcode.dsym")

Source/CarthageKit/Xcode.swift

@@ -6,6 +6,17 @@ import ReactiveSwift
 import ReactiveTask
 import XCDBLD
 
+/// Gets the shared cache file path URL for dependency at pinned version
+/// ~/Library/Caches/org.carthage.CarthageKit/SharedCache/XcodeVersion/DependencyName/Version, exemple :
+/// ~/Library/Caches/org.carthage.CarthageKit/SharedCache/9.4.1_9F2000/SwiftyUserDefaults/3.0.1
+internal func sharedCacheURLByXcodeVersionFor(dependency: Dependency, pinnedVersion: PinnedVersion) -> URL {
+	let sharedCacheURL = Constants.userCachesURL.appendingPathComponent("SharedCache")
+	let sharedCachePerXcode = sharedCacheURL.appendingPathComponent(XcodeVersion.makeString(), isDirectory: true)
+	let sharedCachePerDependency = sharedCachePerXcode.appendingPathComponent(dependency.name, isDirectory: true)
+	let sharedCacheVersioned = sharedCachePerDependency.appendingPathComponent(pinnedVersion.commitish, isDirectory: true)
+	return sharedCacheVersioned
+}
+
 /// Emits the currect Swift version
 internal func swiftVersion(usingToolchain toolchain: String? = nil) -> SignalProducer<String, SwiftVersionError> {
 	return determineSwiftVersion(usingToolchain: toolchain).replayLazily(upTo: 1)
@@ -821,12 +832,20 @@ public func buildInDirectory( // swiftlint:disable:this function_body_length
 				guard let dependency = dependency else {
 					return .empty
 				}
-
-				return createVersionFile(
-					for: dependency.dependency, version: dependency.version,
-					platforms: options.platforms, buildProducts: urls, rootDirectoryURL: rootDirectoryURL
-				)
-				.flatMapError { _ in .empty }
+				var copyToSharedCache = SignalProducer<(URL), CarthageError>.empty
+				if options.cacheBuilds {
+					let sharedCacheURLForDependency = sharedCacheURLByXcodeVersionFor(dependency: dependency.dependency, pinnedVersion: dependency.version)
+					copyToSharedCache = copyFrameworksAndCreateVersionFile(urls, from: rootDirectoryURL, to: sharedCacheURLForDependency,
+																		   projectName: dependency.dependency.name, pinnedVersion: dependency.version, toolchain: options.toolchain)
+				}
+				return copyToSharedCache
+					.flatMap(.merge) { _ -> SignalProducer<(), CarthageError> in
+						return createVersionFile(
+							for: dependency.dependency, version: dependency.version,
+							platforms: options.platforms, buildProducts: urls, rootDirectoryURL: rootDirectoryURL
+						)
+						.flatMapError { _ in .empty }
+					}
 			}
 			// Discard any Success values, since we want to
 			// use our initial value instead of waiting for

Source/XCDBLD/XcodeVersion.swift

@@ -42,4 +42,8 @@ public struct XcodeVersion {
 			}
 			.single()?.value
 	}
+
+	public static func makeString() -> String {
+		return make().map { "\($0.version)_\($0.buildVersion)" } ?? "Unknown"
+	}
 }

Source/carthage/Extensions.swift

@@ -116,6 +116,9 @@ internal struct ProjectEventSink {
 		case let .skippedBuildingCached(dependency):
 			carthage.println(formatting.bullets + "Valid cache found for " + formatting.projectName(dependency.name) + ", skipping build")
 
+		case let .skippedBuildingCopyFromSharedCache(dependency):
+			carthage.println(formatting.bullets + "Copying from shared cache for " + formatting.projectName(dependency.name) + ", skipping build")
+
 		case let .rebuildingCached(dependency):
 			carthage.println(formatting.bullets + "Invalid cache found for " + formatting.projectName(dependency.name)
 				+ ", rebuilding with all downstream dependencies")