diff --git a/Package.swift b/Package.swift index 09054cf7d62..1e6d979dc26 100644 --- a/Package.swift +++ b/Package.swift @@ -258,6 +258,15 @@ let package = Package( ], exclude: ["CMakeLists.txt"] ), + + .target( + name: "PackageFingerprint", + dependencies: [ + "Basics", + "PackageModel", + ], + exclude: ["CMakeLists.txt"] + ), // MARK: Package Manager Functionality @@ -484,6 +493,10 @@ let package = Package( name: "PackageCollectionsTests", dependencies: ["PackageCollections", "SPMTestSupport"] ), + .testTarget( + name: "PackageFingerprintTests", + dependencies: ["PackageFingerprint", "SPMTestSupport"] + ), .testTarget( name: "PackageRegistryTests", dependencies: ["SPMTestSupport", "PackageRegistry"] diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 5b8e30e3c36..004f2de670c 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -15,6 +15,7 @@ add_subdirectory(PackageCollectionsModel) add_subdirectory(PackageCollectionsSigning) add_subdirectory(PackageCollectionsSigningLibc) add_subdirectory(PackageDescription) +add_subdirectory(PackageFingerprint) add_subdirectory(PackageGraph) add_subdirectory(PackageLoading) add_subdirectory(PackageModel) diff --git a/Sources/PackageFingerprint/CMakeLists.txt b/Sources/PackageFingerprint/CMakeLists.txt new file mode 100644 index 00000000000..06dfc0a5570 --- /dev/null +++ b/Sources/PackageFingerprint/CMakeLists.txt @@ -0,0 +1,28 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2021 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +add_library(PackageFingerprint + FilePackageFingerprintStorage.swift + Model.swift + PackageFingerprintStorage.swift) +target_link_libraries(PackageFingerprint PUBLIC + Basics + PackageModel + TSCBasic + TSCUtility) +# NOTE(compnerd) workaround for CMake not setting up include flags yet +set_target_properties(PackageFingerprint PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) + +if(USE_CMAKE_INSTALL) +install(TARGETS PackageFingerprint + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin) +endif() +set_property(GLOBAL APPEND PROPERTY SwiftPM_EXPORTS PackageFingerprint) diff --git a/Sources/PackageFingerprint/FilePackageFingerprintStorage.swift b/Sources/PackageFingerprint/FilePackageFingerprintStorage.swift new file mode 100644 index 00000000000..f1a7e12c25d --- /dev/null +++ b/Sources/PackageFingerprint/FilePackageFingerprintStorage.swift @@ -0,0 +1,190 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2021 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import Basics +import Dispatch +import Foundation +import PackageModel +import TSCBasic +import TSCUtility + +public struct FilePackageFingerprintStorage: PackageFingerprintStorage { + let fileSystem: FileSystem + let directoryPath: AbsolutePath + + private let encoder: JSONEncoder + private let decoder: JSONDecoder + + init(fileSystem: FileSystem, directoryPath: AbsolutePath) { + self.fileSystem = fileSystem + self.directoryPath = directoryPath + + self.encoder = JSONEncoder.makeWithDefaults() + self.decoder = JSONDecoder.makeWithDefaults() + } + + public func get(package: PackageIdentity, + version: Version, + observabilityScope: ObservabilityScope, + callbackQueue: DispatchQueue, + callback: @escaping (Result<[Fingerprint.Kind: Fingerprint], Error>) -> Void) + { + let callback = self.makeAsync(callback, on: callbackQueue) + + do { + let packageFingerprints = try self.withLock { + try self.loadFromDisk(package: package) + } + + guard let fingerprints = packageFingerprints[version] else { + throw PackageFingerprintStorageError.notFound + } + + callback(.success(fingerprints)) + } catch { + callback(.failure(error)) + } + } + + public func put(package: PackageIdentity, + version: Version, + fingerprint: Fingerprint, + observabilityScope: ObservabilityScope, + callbackQueue: DispatchQueue, + callback: @escaping (Result) -> Void) + { + let callback = self.makeAsync(callback, on: callbackQueue) + + do { + try self.withLock { + var packageFingerprints = try self.loadFromDisk(package: package) + + if let existing = packageFingerprints[version]?[fingerprint.origin.kind] { + // Error if we try to write a different fingerprint + guard fingerprint == existing else { + throw PackageFingerprintStorageError.conflict(given: fingerprint, existing: existing) + } + // Don't need to do anything if fingerprints are the same + return + } + + var fingerprints = packageFingerprints.removeValue(forKey: version) ?? [:] + fingerprints[fingerprint.origin.kind] = fingerprint + packageFingerprints[version] = fingerprints + + try self.saveToDisk(package: package, fingerprints: packageFingerprints) + } + callback(.success(())) + } catch { + callback(.failure(error)) + } + } + + private func loadFromDisk(package: PackageIdentity) throws -> PackageFingerprints { + let path = self.directoryPath.appending(component: package.fingerprintFilename) + + guard self.fileSystem.exists(path) else { + return .init() + } + + let buffer = try fileSystem.readFileContents(path).contents + guard buffer.count > 0 else { + return .init() + } + + let container = try self.decoder.decode(StorageModel.Container.self, from: Data(buffer)) + return try container.packageFingerprints() + } + + private func saveToDisk(package: PackageIdentity, fingerprints: PackageFingerprints) throws { + if !self.fileSystem.exists(self.directoryPath) { + try self.fileSystem.createDirectory(self.directoryPath, recursive: true) + } + + let container = StorageModel.Container(fingerprints) + let buffer = try encoder.encode(container) + + let path = self.directoryPath.appending(component: package.fingerprintFilename) + try self.fileSystem.writeFileContents(path, bytes: ByteString(buffer)) + } + + private func withLock(_ body: () throws -> T) throws -> T { + if !self.fileSystem.exists(self.directoryPath) { + try self.fileSystem.createDirectory(self.directoryPath, recursive: true) + } + return try self.fileSystem.withLock(on: self.directoryPath, type: .exclusive, body) + } + + private func makeAsync(_ closure: @escaping (Result) -> Void, on queue: DispatchQueue) -> (Result) -> Void { + { result in queue.async { closure(result) } } + } +} + +private enum StorageModel { + struct Container: Codable { + let versionFingerprints: [String: [String: StoredFingerprint]] + + init(_ versionFingerprints: PackageFingerprints) { + self.versionFingerprints = Dictionary(uniqueKeysWithValues: versionFingerprints.map { version, fingerprints in + let fingerprintByKind: [String: StoredFingerprint] = Dictionary(uniqueKeysWithValues: fingerprints.map { kind, fingerprint in + let origin: String + switch fingerprint.origin { + case .sourceControl(let url): + origin = url.absoluteString + case .registry(let url): + origin = url.absoluteString + } + return (kind.rawValue, StoredFingerprint(origin: origin, fingerprint: fingerprint.value)) + }) + return (version.description, fingerprintByKind) + }) + } + + func packageFingerprints() throws -> PackageFingerprints { + try Dictionary(uniqueKeysWithValues: self.versionFingerprints.map { version, fingerprints in + let fingerprintByKind: [Fingerprint.Kind: Fingerprint] = try Dictionary(uniqueKeysWithValues: fingerprints.map { kind, fingerprint in + guard let kind = Fingerprint.Kind(rawValue: kind) else { + throw SerializationError.unknownKind(kind) + } + guard let originURL = Foundation.URL(string: fingerprint.origin) else { + throw SerializationError.invalidURL(fingerprint.origin) + } + + let origin: Fingerprint.Origin + switch kind { + case .sourceControl: + origin = .sourceControl(originURL) + case .registry: + origin = .registry(originURL) + } + + return (kind, Fingerprint(origin: origin, value: fingerprint.fingerprint)) + }) + return (Version(stringLiteral: version), fingerprintByKind) + }) + } + } + + struct StoredFingerprint: Codable { + let origin: String + let fingerprint: String + } +} + +extension PackageIdentity { + var fingerprintFilename: String { + "\(self.description).json" + } +} + +private enum SerializationError: Error { + case unknownKind(String) + case invalidURL(String) +} diff --git a/Sources/PackageFingerprint/Model.swift b/Sources/PackageFingerprint/Model.swift new file mode 100644 index 00000000000..c6a5352210f --- /dev/null +++ b/Sources/PackageFingerprint/Model.swift @@ -0,0 +1,58 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2021 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import struct Foundation.URL +import TSCUtility + +public struct Fingerprint: Equatable { + public let origin: Origin + public let value: String +} + +public extension Fingerprint { + enum Kind: String, Hashable { + case sourceControl + case registry + } + + enum Origin: Equatable, CustomStringConvertible { + case sourceControl(Foundation.URL) + case registry(Foundation.URL) + + var kind: Fingerprint.Kind { + switch self { + case .sourceControl: + return .sourceControl + case .registry: + return .registry + } + } + + var url: Foundation.URL? { + switch self { + case .sourceControl(let url): + return url + case .registry(let url): + return url + } + } + + public var description: String { + switch self { + case .sourceControl(let url): + return "sourceControl(\(url))" + case .registry(let url): + return "registry(\(url))" + } + } + } +} + +public typealias PackageFingerprints = [Version: [Fingerprint.Kind: Fingerprint]] diff --git a/Sources/PackageFingerprint/PackageFingerprintStorage.swift b/Sources/PackageFingerprint/PackageFingerprintStorage.swift new file mode 100644 index 00000000000..bb43665cc08 --- /dev/null +++ b/Sources/PackageFingerprint/PackageFingerprintStorage.swift @@ -0,0 +1,34 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2021 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import Basics +import Dispatch +import PackageModel +import TSCUtility + +public protocol PackageFingerprintStorage { + func get(package: PackageIdentity, + version: Version, + observabilityScope: ObservabilityScope, + callbackQueue: DispatchQueue, + callback: @escaping (Result<[Fingerprint.Kind: Fingerprint], Error>) -> Void) + + func put(package: PackageIdentity, + version: Version, + fingerprint: Fingerprint, + observabilityScope: ObservabilityScope, + callbackQueue: DispatchQueue, + callback: @escaping (Result) -> Void) +} + +public enum PackageFingerprintStorageError: Error, Equatable { + case conflict(given: Fingerprint, existing: Fingerprint) + case notFound +} diff --git a/Tests/PackageFingerprintTests/FilePackageFingerprintStorageTests.swift b/Tests/PackageFingerprintTests/FilePackageFingerprintStorageTests.swift new file mode 100644 index 00000000000..82fe6213b5a --- /dev/null +++ b/Tests/PackageFingerprintTests/FilePackageFingerprintStorageTests.swift @@ -0,0 +1,142 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2021 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import Basics +import struct Foundation.URL +@testable import PackageFingerprint +import PackageModel +import SPMTestSupport +import TSCBasic +import XCTest + +final class FilePackageFingerprintStorageTests: XCTestCase { + func testHappyCase() throws { + let mockFileSystem = InMemoryFileSystem() + let directoryPath = AbsolutePath("/fingerprints") + let storage = FilePackageFingerprintStorage(fileSystem: mockFileSystem, directoryPath: directoryPath) + let registryURL = Foundation.URL(string: "https://example.packages.com")! + let sourceControlURL = Foundation.URL(string: "https://example.com/mona/LinkedList.git")! + + // Add fingerprints for mona.LinkedList + let package = PackageIdentity.plain("mona.LinkedList") + try storage.put(package: package, version: Version("1.0.0"), fingerprint: .init(origin: .registry(registryURL), value: "checksum-1.0.0")) + try storage.put(package: package, version: Version("1.0.0"), fingerprint: .init(origin: .sourceControl(sourceControlURL), value: "gitHash-1.0.0")) + try storage.put(package: package, version: Version("1.1.0"), fingerprint: .init(origin: .registry(registryURL), value: "checksum-1.1.0")) + // Fingerprint for another package + let otherPackage = PackageIdentity.plain("other.LinkedList") + try storage.put(package: otherPackage, version: Version("1.0.0"), fingerprint: .init(origin: .registry(registryURL), value: "checksum-1.0.0")) + + // A checksum file should have been created for each package + XCTAssertTrue(mockFileSystem.exists(storage.directoryPath.appending(component: package.fingerprintFilename))) + XCTAssertTrue(mockFileSystem.exists(storage.directoryPath.appending(component: otherPackage.fingerprintFilename))) + + // Fingerprints should be saved + do { + let fingerprints = try storage.get(package: package, version: Version("1.0.0")) + XCTAssertEqual(2, fingerprints.count) + + XCTAssertEqual(registryURL, fingerprints[.registry]?.origin.url) + XCTAssertEqual("checksum-1.0.0", fingerprints[.registry]?.value) + + XCTAssertEqual(sourceControlURL, fingerprints[.sourceControl]?.origin.url) + XCTAssertEqual("gitHash-1.0.0", fingerprints[.sourceControl]?.value) + } + + do { + let fingerprints = try storage.get(package: package, version: Version("1.1.0")) + XCTAssertEqual(1, fingerprints.count) + + XCTAssertEqual(registryURL, fingerprints[.registry]?.origin.url) + XCTAssertEqual("checksum-1.1.0", fingerprints[.registry]?.value) + } + + do { + let fingerprints = try storage.get(package: otherPackage, version: Version("1.0.0")) + XCTAssertEqual(1, fingerprints.count) + + XCTAssertEqual(registryURL, fingerprints[.registry]?.origin.url) + XCTAssertEqual("checksum-1.0.0", fingerprints[.registry]?.value) + } + } + + func testNotFound() throws { + let mockFileSystem = InMemoryFileSystem() + let directoryPath = AbsolutePath("/fingerprints") + let storage = FilePackageFingerprintStorage(fileSystem: mockFileSystem, directoryPath: directoryPath) + let registryURL = Foundation.URL(string: "https://example.packages.com")! + + let package = PackageIdentity.plain("mona.LinkedList") + try storage.put(package: package, version: Version("1.0.0"), fingerprint: .init(origin: .registry(registryURL), value: "checksum-1.0.0")) + + // No fingerprints found for the version + XCTAssertThrowsError(try storage.get(package: package, version: Version("1.1.0"))) { error in + guard case PackageFingerprintStorageError.notFound = error else { + return XCTFail("Expected PackageFingerprintStorageError.notFound, got \(error)") + } + } + + // No fingerprints found for the packagge + let otherPackage = PackageIdentity.plain("other.LinkedList") + XCTAssertThrowsError(try storage.get(package: otherPackage, version: Version("1.0.0"))) { error in + guard case PackageFingerprintStorageError.notFound = error else { + return XCTFail("Expected PackageFingerprintStorageError.notFound, got \(error)") + } + } + } + + func testSingleFingerprintPerKind() throws { + let mockFileSystem = InMemoryFileSystem() + let directoryPath = AbsolutePath("/fingerprints") + let storage = FilePackageFingerprintStorage(fileSystem: mockFileSystem, directoryPath: directoryPath) + let registryURL = Foundation.URL(string: "https://example.packages.com")! + + let package = PackageIdentity.plain("mona.LinkedList") + // Write registry checksum for v1.0.0 + try storage.put(package: package, version: Version("1.0.0"), fingerprint: .init(origin: .registry(registryURL), value: "checksum-1.0.0")) + + // Writing for the same version and kind but different checksum should fail + XCTAssertThrowsError(try storage.put(package: package, version: Version("1.0.0"), + fingerprint: .init(origin: .registry(registryURL), value: "checksum-1.0.0-1"))) { error in + guard case PackageFingerprintStorageError.conflict = error else { + return XCTFail("Expected PackageFingerprintStorageError.conflict, got \(error)") + } + } + + // Writing for the same version and kind and same checksum should not fail + XCTAssertNoThrow(try storage.put(package: package, version: Version("1.0.0"), + fingerprint: .init(origin: .registry(registryURL), value: "checksum-1.0.0"))) + } +} + +private extension PackageFingerprintStorage { + func get(package: PackageIdentity, + version: Version) throws -> [Fingerprint.Kind: Fingerprint] { + return try tsc_await { + self.get(package: package, + version: version, + observabilityScope: ObservabilitySystem.NOOP, + callbackQueue: .sharedConcurrent, + callback: $0) + } + } + + func put(package: PackageIdentity, + version: Version, + fingerprint: Fingerprint) throws { + return try tsc_await { + self.put(package: package, + version: version, + fingerprint: fingerprint, + observabilityScope: ObservabilitySystem.NOOP, + callbackQueue: .sharedConcurrent, + callback: $0) + } + } +} diff --git a/Tests/PackageRegistryTests/RegistryManagerTests.swift b/Tests/PackageRegistryTests/RegistryClientTests.swift similarity index 98% rename from Tests/PackageRegistryTests/RegistryManagerTests.swift rename to Tests/PackageRegistryTests/RegistryClientTests.swift index f01f5bbad32..446c30aa4ba 100644 --- a/Tests/PackageRegistryTests/RegistryManagerTests.swift +++ b/Tests/PackageRegistryTests/RegistryClientTests.swift @@ -17,7 +17,7 @@ import SPMTestSupport import TSCBasic import XCTest -final class RegistryManagerTests: XCTestCase { +final class RegistryClientTests: XCTestCase { func testFetchVersions() throws { let registryURL = "https://packages.example.com" let identity = PackageIdentity.plain("mona.LinkedList") @@ -539,8 +539,8 @@ final class RegistryManagerTests: XCTestCase { // MARK - Sugar -extension RegistryClient { - public func fetchVersions(package: PackageIdentity) throws -> [Version] { +private extension RegistryClient { + func fetchVersions(package: PackageIdentity) throws -> [Version] { return try tsc_await { self.fetchVersions( package: package, @@ -551,7 +551,7 @@ extension RegistryClient { } } - public func getAvailableManifests( + func getAvailableManifests( package: PackageIdentity, version: Version ) throws -> [String : ToolsVersion] { @@ -566,7 +566,7 @@ extension RegistryClient { } } - public func getManifestContent( + func getManifestContent( package: PackageIdentity, version: Version, customToolsVersion: ToolsVersion? @@ -583,7 +583,7 @@ extension RegistryClient { } } - public func fetchSourceArchiveChecksum(package: PackageIdentity, version: Version) throws -> String { + func fetchSourceArchiveChecksum(package: PackageIdentity, version: Version) throws -> String { return try tsc_await { self.fetchSourceArchiveChecksum( package: package, @@ -595,7 +595,7 @@ extension RegistryClient { } } - public func downloadSourceArchive( + func downloadSourceArchive( package: PackageIdentity, version: Version, fileSystem: FileSystem, @@ -619,7 +619,7 @@ extension RegistryClient { } } - public func lookupIdentities(url: Foundation.URL) throws -> Set { + func lookupIdentities(url: Foundation.URL) throws -> Set { return try tsc_await { self.lookupIdentities( url: url,