diff --git a/Package.swift b/Package.swift index 5d336ae..1651845 100644 --- a/Package.swift +++ b/Package.swift @@ -47,11 +47,13 @@ let package = Package( "PromiseKit", .product(name: "PMKFoundation", package: "Foundation"), "SwiftSoup", - "Unxip", "Version", .product(name: "XCModel", package: "data"), "Rainbow", "Yams" + ], + resources: [ + .copy("Resources/unxip"), ]), .testTarget( name: "XcodesKitTests", @@ -62,7 +64,6 @@ let package = Package( resources: [ .copy("Fixtures"), ]), - .target(name: "Unxip"), .target( name: "AppleAPI", dependencies: [ diff --git a/Sources/Unxip/Unxip.swift b/Sources/Unxip/Unxip.swift deleted file mode 100644 index d45bc48..0000000 --- a/Sources/Unxip/Unxip.swift +++ /dev/null @@ -1,535 +0,0 @@ -import Compression -import Foundation - -// From: https://github.com/saagarjha/unxip -// License: GNU Lesser General Public License v3.0 - -extension RandomAccessCollection { - subscript(fromOffset fromOffset: Int = 0, toOffset toOffset: Int? = nil) -> SubSequence { - let toOffset = toOffset ?? count - return self[index(startIndex, offsetBy: fromOffset).. SubSequence { - let base = index(startIndex, offsetBy: fromOffset) - return self[base.. { - let batchSize: Int - var operations = [@Sendable () async throws -> TaskResult]() - - var results: AsyncStream { - AsyncStream(bufferingPolicy: .bufferingOldest(batchSize)) { continuation in - Task { - try await withThrowingTaskGroup(of: (Int, TaskResult).self) { group in - var queueIndex = 0 - var dequeIndex = 0 - var pending = [Int: TaskResult]() - while dequeIndex < operations.count { - if queueIndex - dequeIndex < batchSize, - queueIndex < operations.count - { - let _queueIndex = queueIndex - group.addTask { - let queueIndex = _queueIndex - return await (queueIndex, try operations[queueIndex]()) - } - queueIndex += 1 - } else { - let (index, result) = try await group.next()! - pending[index] = result - if index == dequeIndex { - while let result = pending[dequeIndex] { - await continuation.yieldWithBackoff(result) - pending.removeValue(forKey: dequeIndex) - dequeIndex += 1 - } - } - } - } - continuation.finish() - } - } - } - } - - init(batchSize: Int = ProcessInfo.processInfo.activeProcessorCount) { - self.batchSize = batchSize - } - - mutating func addTask(operation: @escaping @Sendable () async throws -> TaskResult) { - operations.append(operation) - } - - mutating func addRunningTask(operation: @escaping @Sendable () async -> TaskResult) -> Task { - let task = Task { - await operation() - } - operations.append({ - await task.value - }) - return task - } -} - -final class Chunk: Sendable { - let buffer: UnsafeBufferPointer - let owned: Bool - - init(buffer: UnsafeBufferPointer, owned: Bool) { - self.buffer = buffer - self.owned = owned - } - - deinit { - if owned { - buffer.deallocate() - } - } -} - -struct File { - let dev: Int - let ino: Int - let mode: Int - let name: String - var data = [UnsafeBufferPointer]() - // For keeping the data alive - var chunks = [Chunk]() - - struct Identifier: Hashable { - let dev: Int - let ino: Int - } - - var identifier: Identifier { - Identifier(dev: dev, ino: ino) - } - - func compressedData() async -> [UInt8]? { - let blockSize = 64 << 10 // LZFSE with 64K block size - var _data = [UInt8]() - _data.reserveCapacity(self.data.map(\.count).reduce(0, +)) - let data = self.data.reduce(into: _data, +=) - var compressionStream = ConcurrentStream<[UInt8]?>() - var position = data.startIndex - - while position < data.endIndex { - let _position = position - compressionStream.addTask { - try Task.checkCancellation() - let position = _position - let end = min(position + blockSize, data.endIndex) - let data = [UInt8](unsafeUninitializedCapacity: (end - position) + (end - position) / 16) { buffer, count in - data[position...size - let size = tableSize + chunks.map(\.count).reduce(0, +) - guard size < data.count else { - return nil - } - - return [UInt8](unsafeUninitializedCapacity: size) { buffer, count in - var position = tableSize - - func writePosition(toTableIndex index: Int) { - precondition(position < UInt32.max) - for i in 0...size { - buffer[index * MemoryLayout.size + i] = UInt8(position >> (i * 8) & 0xff) - } - } - - writePosition(toTableIndex: 0) - for (index, chunk) in zip(1..., chunks) { - _ = UnsafeMutableBufferPointer(rebasing: buffer.suffix(from: position)).initialize(from: chunk) - position += chunk.count - writePosition(toTableIndex: index) - } - count = size - } - } - - func write(compressedData data: [UInt8], toDescriptor descriptor: CInt) -> Bool { - let uncompressedSize = self.data.map(\.count).reduce(0, +) - let attribute = - "cmpf".utf8.reversed() // magic - + [0x0c, 0x00, 0x00, 0x00] // LZFSE, 64K chunks - + ([ - (uncompressedSize >> 0) & 0xff, - (uncompressedSize >> 8) & 0xff, - (uncompressedSize >> 16) & 0xff, - (uncompressedSize >> 24) & 0xff, - (uncompressedSize >> 32) & 0xff, - (uncompressedSize >> 40) & 0xff, - (uncompressedSize >> 48) & 0xff, - (uncompressedSize >> 56) & 0xff, - ].map(UInt8.init) as [UInt8]) - - guard fsetxattr(descriptor, "com.apple.decmpfs", attribute, attribute.count, 0, XATTR_SHOWCOMPRESSION) == 0 else { - return false - } - - let resourceForkDescriptor = open(name + _PATH_RSRCFORKSPEC, O_WRONLY | O_CREAT, 0o666) - guard resourceForkDescriptor >= 0 else { - return false - } - defer { - close(resourceForkDescriptor) - } - - var written: Int - repeat { - // TODO: handle partial writes smarter - written = pwrite(resourceForkDescriptor, data, data.count, 0) - guard written >= 0 else { - return false - } - } while written != data.count - - guard fchflags(descriptor, UInt32(UF_COMPRESSED)) == 0 else { - return false - } - - return true - } -} - -extension option { - init(name: StaticString, has_arg: CInt, flag: UnsafeMutablePointer?, val: StringLiteralType) { - let _option = name.withUTF8Buffer { - $0.withMemoryRebound(to: CChar.self) { - option(name: $0.baseAddress, has_arg: has_arg, flag: flag, val: CInt(UnicodeScalar(val)!.value)) - } - } - self = _option - } -} - -public struct UnxipOptions { - var input: URL - var output: URL? - var compress: Bool = true - - public init(input: URL, output: URL) { - self.input = input - self.output = output - } -} - -@available(macOS 11.0, *) -public struct Unxip { - let options: UnxipOptions - - public init(options: UnxipOptions) { - self.options = options - } - - func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { - defer { - buffer = buffer[fromOffset: MemoryLayout.size] - } - var result: Integer = 0 - var iterator = buffer.makeIterator() - for _ in 0...size { - result <<= 8 - result |= Integer(iterator.next()!) - } - return result - } - - func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { - var remaining = content[fromOffset: 4] - let chunkSize = read(UInt64.self, from: &remaining) - var decompressedSize: UInt64 = 0 - - var chunkStream = ConcurrentStream() - - repeat { - decompressedSize = read(UInt64.self, from: &remaining) - let compressedSize = read(UInt64.self, from: &remaining) - let _remaining = remaining - let _decompressedSize = decompressedSize - - chunkStream.addTask { - let remaining = _remaining - let decompressedSize = _decompressedSize - - if compressedSize == chunkSize { - return Chunk(buffer: UnsafeBufferPointer(rebasing: remaining[fromOffset: 0, size: Int(compressedSize)]), owned: false) - } else { - let magic = [0xfd] + "7zX".utf8 - precondition(remaining.prefix(magic.count).elementsEqual(magic)) - let buffer = UnsafeMutableBufferPointer.allocate(capacity: Int(decompressedSize)) - precondition(compression_decode_buffer(buffer.baseAddress!, buffer.count, UnsafeBufferPointer(rebasing: remaining).baseAddress!, Int(compressedSize), nil, COMPRESSION_LZMA) == decompressedSize) - return Chunk(buffer: UnsafeBufferPointer(buffer), owned: true) - } - } - remaining = remaining[fromOffset: Int(compressedSize)] - } while decompressedSize == chunkSize - - return chunkStream - } - - func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { - AsyncStream(bufferingPolicy: .bufferingOldest(ProcessInfo.processInfo.activeProcessorCount)) { continuation in - Task { - var iterator = chunkStream.makeAsyncIterator() - var chunk = try! await iterator.next()! - var position = 0 - - func read(size: Int) async -> [UInt8] { - var result = [UInt8]() - while result.count < size { - if position >= chunk.buffer.endIndex { - chunk = try! await iterator.next()! - position = 0 - } - result.append(chunk.buffer[chunk.buffer.startIndex + position]) - position += 1 - } - return result - } - - func readOctal(from bytes: [UInt8]) -> Int { - Int(String(data: Data(bytes), encoding: .utf8)!, radix: 8)! - } - - while true { - let magic = await read(size: 6) - // Yes, cpio.h really defines this global macro - precondition(magic.elementsEqual(MAGIC.utf8)) - let dev = readOctal(from: await read(size: 6)) - let ino = readOctal(from: await read(size: 6)) - let mode = readOctal(from: await read(size: 6)) - let _ = await read(size: 6) // uid - let _ = await read(size: 6) // gid - let _ = await read(size: 6) // nlink - let _ = await read(size: 6) // rdev - let _ = await read(size: 11) // mtime - let namesize = readOctal(from: await read(size: 6)) - var filesize = readOctal(from: await read(size: 11)) - let name = String(cString: await read(size: namesize)) - var file = File(dev: dev, ino: ino, mode: mode, name: name) - - while filesize > 0 { - if position >= chunk.buffer.endIndex { - chunk = try! await iterator.next()! - position = 0 - } - let size = min(filesize, chunk.buffer.endIndex - position) - file.chunks.append(chunk) - file.data.append(UnsafeBufferPointer(rebasing: chunk.buffer[fromOffset: position, size: size])) - filesize -= size - position += size - } - - guard file.name != "TRAILER!!!" else { - continuation.finish() - return - } - - await continuation.yieldWithBackoff(file) - } - } - } - } - - public func parseContent(_ content: UnsafeBufferPointer) async { - var taskStream = ConcurrentStream(batchSize: 64) // Worst case, should allow for files up to 64 * 16MB = 1GB - var hardlinks = [File.Identifier: (String, Task)]() - var directories = [Substring: Task]() - for await file in files(in: chunks(from: content).results) { - @Sendable - func warn(_ result: CInt, _ operation: String) { - if result != 0 { - perror("\(operation) \(file.name) failed") - } - } - - // The assumption is that all directories are provided without trailing slashes - func parentDirectory(of path: S) -> S.SubSequence { - path[.. Task? { - directories[parentDirectory(of: file.name)] ?? directories[String(parentDirectory(of: file.name))[...]] - } - - @Sendable - func setStickyBit(on file: File) { - if file.mode & Int(C_ISVTX) != 0 { - warn(chmod(file.name, mode_t(file.mode)), "Setting sticky bit on") - } - } - - if file.name == "." { - continue - } - - if let (original, originalTask) = hardlinks[file.identifier] { - let task = parentDirectoryTask(for: file) - assert(task != nil, file.name) - _ = taskStream.addRunningTask { - _ = await (originalTask.value, task?.value) - - warn(link(original, file.name), "linking") - } - continue - } - - // The types we care about, anyways - let typeMask = Int(C_ISLNK | C_ISDIR | C_ISREG) - switch CInt(file.mode & typeMask) { - case C_ISLNK: - let task = parentDirectoryTask(for: file) - assert(task != nil, file.name) - _ = taskStream.addRunningTask { - await task?.value - - warn(symlink(String(data: Data(file.data.map(Array.init).reduce([], +)), encoding: .utf8), file.name), "symlinking") - setStickyBit(on: file) - } - case C_ISDIR: - let task = parentDirectoryTask(for: file) - assert(task != nil || parentDirectory(of: file.name) == ".", file.name) - directories[file.name[...]] = taskStream.addRunningTask { - await task?.value - - warn(mkdir(file.name, mode_t(file.mode & 0o777)), "creating directory at") - setStickyBit(on: file) - } - case C_ISREG: - let task = parentDirectoryTask(for: file) - assert(task != nil, file.name) - hardlinks[file.identifier] = ( - file.name, - taskStream.addRunningTask { - await task?.value - let compressedData = options.compress ? await file.compressedData() : nil - - let fd = open(file.name, O_CREAT | O_WRONLY, mode_t(file.mode & 0o777)) - if fd < 0 { - warn(fd, "creating file at") - return - } - defer { - warn(close(fd), "closing") - setStickyBit(on: file) - } - - if let compressedData = compressedData, - file.write(compressedData: compressedData, toDescriptor: fd) - { - return - } - - // pwritev requires the vector count to be positive - if file.data.count == 0 { - return - } - - var vector = file.data.map { - iovec(iov_base: UnsafeMutableRawPointer(mutating: $0.baseAddress), iov_len: $0.count) - } - let total = file.data.map(\.count).reduce(0, +) - var written = 0 - - repeat { - // TODO: handle partial writes smarter - written = pwritev(fd, &vector, CInt(vector.count), 0) - if written < 0 { - warn(-1, "writing chunk to") - break - } - } while written != total - } - ) - default: - fatalError("\(file.name) with \(file.mode) is a type that is unhandled") - } - } - - // Run through any stragglers - for await _ in taskStream.results { - } - } - - func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { - precondition(file.starts(with: "xar!".utf8)) // magic - var header = file[4...] - let headerSize = read(UInt16.self, from: &header) - precondition(read(UInt16.self, from: &header) == 1) // version - let tocCompressedSize = read(UInt64.self, from: &header) - let tocDecompressedSize = read(UInt64.self, from: &header) - _ = read(UInt32.self, from: &header) // checksum - - let toc = [UInt8](unsafeUninitializedCapacity: Int(tocDecompressedSize)) { buffer, count in - let zlibSkip = 2 // Apple's decoder doesn't want to see CMF/FLG (see RFC 1950) - count = compression_decode_buffer(buffer.baseAddress!, Int(tocDecompressedSize), file.baseAddress! + Int(headerSize) + zlibSkip, Int(tocCompressedSize) - zlibSkip, nil, COMPRESSION_ZLIB) - precondition(count == Int(tocDecompressedSize)) - } - - let document = try! XMLDocument(data: Data(toc)) - let content = try! document.nodes(forXPath: "xar/toc/file").first { - try! $0.nodes(forXPath: "name").first!.stringValue! == "Content" - }! - let contentOffset = Int(try! content.nodes(forXPath: "data/offset").first!.stringValue!)! - let contentSize = Int(try! content.nodes(forXPath: "data/length").first!.stringValue!)! - let contentBase = Int(headerSize) + Int(tocCompressedSize) + contentOffset - - let slice = file[fromOffset: contentBase, size: contentSize] - precondition(slice.starts(with: "pbzx".utf8)) - return UnsafeBufferPointer(rebasing: slice) - } - - public func run() async throws { - let handle = try FileHandle(forReadingFrom: options.input) - try handle.seekToEnd() - let length = Int(try handle.offset()) - let file = UnsafeBufferPointer(start: mmap(nil, length, PROT_READ, MAP_PRIVATE, handle.fileDescriptor, 0).bindMemory(to: UInt8.self, capacity: length), count: length) - precondition(UnsafeMutableRawPointer(mutating: file.baseAddress) != MAP_FAILED) - defer { - munmap(UnsafeMutableRawPointer(mutating: file.baseAddress), length) - } - - if let output = options.output { - guard chdir(output.path) == 0 else { - fputs("Failed to access output directory at \(output.path): \(String(cString: strerror(errno)))", stderr) - exit(EXIT_FAILURE) - } - } - - await parseContent(locateContent(in: file)) - } -} diff --git a/Sources/XcodesKit/Environment.swift b/Sources/XcodesKit/Environment.swift index 393e19c..6cc697d 100644 --- a/Sources/XcodesKit/Environment.swift +++ b/Sources/XcodesKit/Environment.swift @@ -25,6 +25,21 @@ public var Current = Environment() public struct Shell { public var unxip: (URL) -> Promise = { Process.run(Path.root.usr.bin.xip, workingDirectory: $0.deletingLastPathComponent(), "--expand", "\($0.path)") } + public var unxipExperimental: (URL) -> Promise = { url in + let workingDir = url.deletingLastPathComponent() + + // 1) Try bundled unxip first + if let bundledURL = Bundle.module.url(forResource: "unxip", withExtension: nil) { + guard let bundledPath = Path(url: bundledURL) else { + return Process.run(Path.root.usr.bin.xip, workingDirectory: workingDir, "--expand", "\(url.path)") + } + return Process.run(bundledPath, workingDirectory: workingDir, "\(url.path)") + } + + Current.logging.log("Can't find unxip bundle path".black.onYellow) + // 2) Fallback to system xip --expand + return Process.run(Path.root.usr.bin.xip, workingDirectory: workingDir, "--expand", "\(url.path)") + } public var mountDmg: (URL) -> Promise = { Process.run(Path.root.usr.bin.join("hdiutil"), "attach", "-nobrowse", "-plist", $0.path) } public var unmountDmg: (URL) -> Promise = { Process.run(Path.root.usr.bin.join("hdiutil"), "detach", $0.path) } public var expandPkg: (URL, URL) -> Promise = { Process.run(Path.root.usr.sbin.join("pkgutil"), "--expand", $0.path, $1.path) } @@ -328,3 +343,4 @@ public struct Keychain { try remove(key) } } + diff --git a/Sources/XcodesKit/Process.swift b/Sources/XcodesKit/Process.swift index dd79217..eb57d52 100644 --- a/Sources/XcodesKit/Process.swift +++ b/Sources/XcodesKit/Process.swift @@ -22,20 +22,58 @@ extension Process { @discardableResult static func run(_ executable: URL, workingDirectory: URL? = nil, input: String? = nil, _ arguments: [String]) -> Promise { - let process = Process() - process.currentDirectoryURL = workingDirectory ?? executable.deletingLastPathComponent() - process.executableURL = executable - process.arguments = arguments - if let input = input { - let inputPipe = Pipe() - process.standardInput = inputPipe.fileHandleForReading - inputPipe.fileHandleForWriting.write(Data(input.utf8)) - inputPipe.fileHandleForWriting.closeFile() - } - return process.launch(.promise).map { std in - let output = String(data: std.out.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" - let error = String(data: std.err.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" - return (process.terminationStatus, output, error) + return Promise { seal in + let process = Process() + process.currentDirectoryURL = workingDirectory ?? executable.deletingLastPathComponent() + process.executableURL = executable + process.arguments = arguments + + let outPipe = Pipe() + let errPipe = Pipe() + process.standardOutput = outPipe + process.standardError = errPipe + var output = Data() + var error = Data() + + outPipe.fileHandleForReading.readabilityHandler = { handle in + let data = handle.availableData + if !data.isEmpty { + output.append(data) + } + } + errPipe.fileHandleForReading.readabilityHandler = { handle in + let data = handle.availableData + if !data.isEmpty { + error.append(data) + } + } + + if let input = input { + let inputPipe = Pipe() + process.standardInput = inputPipe + if let inputData = input.data(using: .utf8) { + inputPipe.fileHandleForWriting.write(inputData) + } + inputPipe.fileHandleForWriting.closeFile() + } + + do { + try process.run() + } catch { + outPipe.fileHandleForReading.readabilityHandler = nil + errPipe.fileHandleForReading.readabilityHandler = nil + seal.reject(error) + return + } + + DispatchQueue.global(qos: .userInitiated).async { + process.waitUntilExit() + outPipe.fileHandleForReading.readabilityHandler = nil + errPipe.fileHandleForReading.readabilityHandler = nil + let outputString = String(data: output, encoding: .utf8) ?? "" + let errorString = String(data: error, encoding: .utf8) ?? "" + seal.fulfill((process.terminationStatus, outputString, errorString)) + } } } } diff --git a/Sources/XcodesKit/Resources/unxip b/Sources/XcodesKit/Resources/unxip new file mode 100755 index 0000000..4f2b5d6 Binary files /dev/null and b/Sources/XcodesKit/Resources/unxip differ diff --git a/Sources/XcodesKit/XcodeInstaller.swift b/Sources/XcodesKit/XcodeInstaller.swift index 44747c6..90c490f 100644 --- a/Sources/XcodesKit/XcodeInstaller.swift +++ b/Sources/XcodesKit/XcodeInstaller.swift @@ -5,7 +5,6 @@ import AppleAPI import Version import LegibleError import Rainbow -import Unxip /// Downloads and installs Xcodes public final class XcodeInstaller { @@ -624,19 +623,16 @@ public final class XcodeInstaller { Current.logging.log(InstallationStep.unarchiving(experimentalUnxip: experimentalUnxip).description) if experimentalUnxip, #available(macOS 11, *) { - return Promise { seal in - Task.detached { - let output = source.deletingLastPathComponent() - let options = UnxipOptions(input: source, output: output) - - do { - try await Unxip(options: options).run() - seal.resolve(.fulfilled(())) - } catch { - seal.reject(error) + + return Current.shell.unxipExperimental(source) + .recover { (error) throws -> Promise in + if case Process.PMKError.execution(_, _, let standardError) = error, + standardError?.contains("damaged and can’t be expanded") == true { + throw Error.damagedXIP(url: source) } + throw error } - } + .map { _ in () } } return Current.shell.unxip(source)