diff --git a/Package.swift b/Package.swift index 379133a5..d9081b49 100644 --- a/Package.swift +++ b/Package.swift @@ -19,7 +19,10 @@ let targets: [PackageDescription.Target] = [ path: "Sources/System"), .target( name: "SystemInternals", - dependencies: ["CSystem"]), + dependencies: ["CSystem"], + swiftSettings: [ + .define("ENABLE_MOCKING") + ]), .target( name: "CSystem", dependencies: []), diff --git a/README.md b/README.md index 4b36b9d4..dd8c634a 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ try fd.closeAfter { ## Adding `SystemPackage` as a Dependency -To use the `SystemPackage` library in a SwiftPM project, +To use the `SystemPackage` library in a SwiftPM project, add the following line to the dependencies in your `Package.swift` file: ```swift diff --git a/Sources/SystemInternals/Exports.swift b/Sources/SystemInternals/Exports.swift index c45bf329..5fec9046 100644 --- a/Sources/SystemInternals/Exports.swift +++ b/Sources/SystemInternals/Exports.swift @@ -35,50 +35,6 @@ public var system_errno: CInt { } #endif -public func system_open(_ path: UnsafePointer, _ oflag: Int32) -> CInt { - open(path, oflag) -} - -public func system_open( - _ path: UnsafePointer, _ oflag: Int32, _ mode: mode_t -) -> CInt { - open(path, oflag, mode) -} - -public func system_close(_ fd: Int32) -> Int32 { - close(fd) -} - -public func system_read( - _ fd: Int32, _ buf: UnsafeMutableRawPointer!, _ nbyte: Int -) -> Int { - read(fd, buf, nbyte) -} - -public func system_pread( - _ fd: Int32, _ buf: UnsafeMutableRawPointer!, _ nbyte: Int, _ offset: off_t -) -> Int { - pread(fd, buf, nbyte, offset) -} - -public func system_lseek( - _ fd: Int32, _ off: off_t, _ whence: Int32 -) -> off_t { - lseek(fd, off, whence) -} - -public func system_write( - _ fd: Int32, _ buf: UnsafeRawPointer!, _ nbyte: Int -) -> Int { - write(fd, buf, nbyte) -} - -public func system_pwrite( - _ fd: Int32, _ buf: UnsafeRawPointer!, _ nbyte: Int, _ offset: off_t -) -> Int { - pwrite(fd, buf, nbyte, offset) -} - // MARK: C stdlib decls public func system_strerror(_ __errnum: Int32) -> UnsafeMutablePointer! { diff --git a/Sources/SystemInternals/Mocking.swift b/Sources/SystemInternals/Mocking.swift new file mode 100644 index 00000000..74a67a01 --- /dev/null +++ b/Sources/SystemInternals/Mocking.swift @@ -0,0 +1,160 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2020 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information +*/ + +// Syscall mocking support. +// +// NOTE: This is currently the bare minimum needed for System's testing purposes, though we do +// eventually want to expose some solution to users. +// +// Mocking is contextual, accessible through MockingDriver.withMockingEnabled. Mocking +// state, including whether it is enabled, is stored in thread-local storage. Mocking is only +// enabled in testing builds of System currently, to minimize runtime overhead of release builds. +// + +public struct Trace { + public struct Entry: Hashable { + var name: String + var arguments: [AnyHashable] + + public init(name: String, _ arguments: [AnyHashable]) { + self.name = name + self.arguments = arguments + } + } + + private var entries: [Entry] = [] + private var firstEntry: Int = 0 + + public var isEmpty: Bool { firstEntry >= entries.count } + + public mutating func dequeue() -> Entry? { + guard !self.isEmpty else { return nil } + defer { firstEntry += 1 } + return entries[firstEntry] + } + + internal mutating func add(_ e: Entry) { + entries.append(e) + } + + public mutating func clear() { entries.removeAll() } +} + +// TODO: Track +public struct WriteBuffer { + public var enabled: Bool = false + + private var buffer: [UInt8] = [] + private var chunkSize: Int? = nil + + internal mutating func write(_ buf: UnsafeRawBufferPointer) -> Int { + guard enabled else { return 0 } + let chunk = chunkSize ?? buf.count + buffer.append(contentsOf: buf.prefix(chunk)) + return chunk + } + + public var contents: [UInt8] { buffer } +} + +public enum ForceErrno { + case none + case always(errno: CInt) + + case counted(errno: CInt, count: Int) +} + +// Provide access to the driver, context, and trace stack of mocking +public class MockingDriver { + // Whether to bypass this shim and go straight to the syscall + public var enableMocking = false + + // Record syscalls and their arguments + public var trace = Trace() + + // Mock errors inside syscalls + public var forceErrno = ForceErrno.none + + // A buffer to put `write` bytes into + public var writeBuffer = WriteBuffer() +} + +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) +import Darwin +#elseif os(Linux) || os(FreeBSD) || os(Android) +import Glibc +#else +#error("Unsupported Platform") +#endif + +internal let key: pthread_key_t = { + var raw = pthread_key_t() + func releaseObject(_ raw: UnsafeMutableRawPointer) -> () { + Unmanaged.fromOpaque(raw).release() + } + guard 0 == pthread_key_create(&raw, releaseObject) else { + fatalError("Unable to create key") + } + // TODO: All threads are sharing the same object; this is wrong + let object = MockingDriver() + + guard 0 == pthread_setspecific(raw, Unmanaged.passRetained(object).toOpaque()) else { + fatalError("Unable to set TLSData") + } + return raw +}() + +internal var currentMockingDriver: MockingDriver { + #if !ENABLE_MOCKING + fatalError("Contextual mocking in non-mocking build") + #endif + + // TODO: Do we need a lazy initialization check here? + return Unmanaged.fromOpaque(pthread_getspecific(key)!).takeUnretainedValue() +} + +extension MockingDriver { + /// Whether mocking is enabled on this thread + public static var enabled: Bool { mockingEnabled } + + /// Enables mocking for the duration of `f` with a clean trace queue + /// Restores prior mocking status and trace queue after execution + public static func withMockingEnabled( + _ f: (MockingDriver) throws -> () + ) rethrows { + let priorMocking = currentMockingDriver.enableMocking + defer { + currentMockingDriver.enableMocking = priorMocking + } + currentMockingDriver.enableMocking = true + + let oldTrace = currentMockingDriver.trace + defer { currentMockingDriver.trace = oldTrace } + + currentMockingDriver.trace.clear() + return try f(currentMockingDriver) + } +} + +// Check TLS for mocking +@inline(never) +private var contextualMockingEnabled: Bool { + return currentMockingDriver.enableMocking +} + +@inline(__always) +internal var mockingEnabled: Bool { + // Fast constant-foldable check for release builds + #if !ENABLE_MOCKING + return false + #endif + + return contextualMockingEnabled +} + diff --git a/Sources/SystemInternals/Syscalls.swift b/Sources/SystemInternals/Syscalls.swift new file mode 100644 index 00000000..d08fc6db --- /dev/null +++ b/Sources/SystemInternals/Syscalls.swift @@ -0,0 +1,177 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2020 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information +*/ + +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) +import Darwin +#elseif os(Linux) || os(FreeBSD) || os(Android) +import Glibc +#else +#error("Unsupported Platform") +#endif + +// Strip the mock_system prefix and the arg list suffix +private func originalSyscallName(_ s: String) -> String { + precondition(s.starts(with: "mock_system_")) + return String(s.dropFirst("mock_system_".count).prefix { $0.isLetter }) +} + +private func mockImpl( + name: String, + _ args: [AnyHashable] +) -> CInt { + let origName = originalSyscallName(name) + let driver = currentMockingDriver + driver.trace.add(Trace.Entry(name: origName, args)) + + switch driver.forceErrno { + case .none: break + case .always(let e): + errno = e + return -1 + case .counted(let e, let count): + assert(count >= 1) + errno = e + driver.forceErrno = count > 1 ? .counted(errno: e, count: count-1) : .none + return -1 + } + + return 0 +} + +private func mock( + name: String = #function, _ args: AnyHashable... +) -> CInt { + precondition(mockingEnabled) + return mockImpl(name: name, args) +} +private func mockInt( + name: String = #function, _ args: AnyHashable... +) -> Int { + Int(mockImpl(name: name, args)) +} + +private func mockOffT( + name: String = #function, _ args: AnyHashable... +) -> off_t { + off_t(mockImpl(name: name, args)) +} + +// Interacting with the mocking system, tracing, etc., is a potentially significant +// amount of code size, so we hand outline that code for every syscall + +// open +public func system_open(_ path: UnsafePointer, _ oflag: Int32) -> CInt { + if _fastPath(!mockingEnabled) { return open(path, oflag) } + return mock_system_open(path, oflag) +} + +@inline(never) +private func mock_system_open(_ path: UnsafePointer, _ oflag: Int32) -> CInt { + mock(String(cString: path), oflag) +} + +public func system_open( + _ path: UnsafePointer, _ oflag: Int32, _ mode: mode_t +) -> CInt { + if _fastPath(!mockingEnabled) { return open(path, oflag, mode) } + return mock_system_open(path, oflag, mode) +} + +@inline(never) +private func mock_system_open( + _ path: UnsafePointer, _ oflag: Int32, _ mode: mode_t +) -> CInt { + mock(String(cString: path), oflag, mode) +} + +// close +public func system_close(_ fd: Int32) -> Int32 { + if _fastPath(!mockingEnabled) { return close(fd) } + return mock_system_close(fd) +} + +@inline(never) +private func mock_system_close(_ fd: Int32) -> Int32 { + mock(fd) +} + +// read +public func system_read( + _ fd: Int32, _ buf: UnsafeMutableRawPointer!, _ nbyte: Int +) -> Int { + if _fastPath(!mockingEnabled) { return read(fd, buf, nbyte) } + return mock_system_read(fd, buf, nbyte) +} + +@inline(never) +private func mock_system_read( + _ fd: Int32, _ buf: UnsafeMutableRawPointer!, _ nbyte: Int +) -> Int { + mockInt(fd, buf, nbyte) +} + +// pread +public func system_pread( + _ fd: Int32, _ buf: UnsafeMutableRawPointer!, _ nbyte: Int, _ offset: off_t +) -> Int { + if _fastPath(!mockingEnabled) { return pread(fd, buf, nbyte, offset) } + return mock_system_pread(fd, buf, nbyte, offset) +} + +@inline(never) +private func mock_system_pread( + _ fd: Int32, _ buf: UnsafeMutableRawPointer!, _ nbyte: Int, _ offset: off_t +) -> Int { + mockInt(fd, buf, nbyte, offset) +} + +// lseek +public func system_lseek( + _ fd: Int32, _ off: off_t, _ whence: Int32 +) -> off_t { + if _fastPath(!mockingEnabled) { return lseek(fd, off, whence) } + return mock_system_lseek(fd, off, whence) +} + +@inline(never) +private func mock_system_lseek( + _ fd: Int32, _ off: off_t, _ whence: Int32 +) -> off_t { + mockOffT(fd, off, whence) +} + +// write +public func system_write( + _ fd: Int32, _ buf: UnsafeRawPointer!, _ nbyte: Int +) -> Int { + if _fastPath(!mockingEnabled) { return write(fd, buf, nbyte) } + return mock_system_write(fd, buf, nbyte) +} + +@inline(never) +private func mock_system_write( + _ fd: Int32, _ buf: UnsafeRawPointer!, _ nbyte: Int +) -> Int { + mockInt(fd, buf, nbyte) +} + +// pwrite +public func system_pwrite( + _ fd: Int32, _ buf: UnsafeRawPointer!, _ nbyte: Int, _ offset: off_t +) -> Int { + if _fastPath(!mockingEnabled) { return pwrite(fd, buf, nbyte, offset) } + return mock_system_pwrite(fd, buf, nbyte, offset) +} + +@inline(never) +private func mock_system_pwrite( + _ fd: Int32, _ buf: UnsafeRawPointer!, _ nbyte: Int, _ offset: off_t +) -> Int { + mockInt(fd, buf, nbyte, offset) +} diff --git a/Tests/SystemTests/FileOperationsTest.swift b/Tests/SystemTests/FileOperationsTest.swift index d3f74162..b3932ddf 100644 --- a/Tests/SystemTests/FileOperationsTest.swift +++ b/Tests/SystemTests/FileOperationsTest.swift @@ -12,6 +12,104 @@ import SystemPackage // @available(macOS 10.16, iOS 14.0, watchOS 7.0, tvOS 14.0, *) final class FileOperationsTest: XCTestCase { - // TODO: Mocking based testing + func testSyscalls() { + let fd = FileDescriptor(rawValue: 1) + + let rawBuf = UnsafeMutableRawBufferPointer.allocate(byteCount: 100, alignment: 4) + defer { rawBuf.deallocate() } + let bufAddr = rawBuf.baseAddress + let rawFD = fd.rawValue + let bufCount = rawBuf.count + let writeBuf = UnsafeRawBufferPointer(rawBuf) + let writeBufAddr = writeBuf.baseAddress + + let syscallTestCases: Array = [ + MockTestCase(name: "open", "a path", O_RDWR | O_APPEND, interruptable: true) { + retryOnInterrupt in + _ = try FileDescriptor.open( + "a path", .readWrite, options: [.append], retryOnInterrupt: retryOnInterrupt) + }, + + MockTestCase(name: "open", "a path", O_WRONLY | O_CREAT | O_APPEND, 0o777, interruptable: true) { + retryOnInterrupt in + _ = try FileDescriptor.open( + "a path", .writeOnly, options: [.create, .append], + permissions: [.groupReadWriteExecute, .ownerReadWriteExecute, .otherReadWriteExecute], + retryOnInterrupt: retryOnInterrupt) + }, + + MockTestCase(name: "read", rawFD, bufAddr, bufCount, interruptable: true) { + retryOnInterrupt in + _ = try fd.read(into: rawBuf, retryOnInterrupt: retryOnInterrupt) + }, + + MockTestCase(name: "pread", rawFD, bufAddr, bufCount, 5, interruptable: true) { + retryOnInterrupt in + _ = try fd.read(fromAbsoluteOffset: 5, into: rawBuf, retryOnInterrupt: retryOnInterrupt) + }, + + MockTestCase(name: "lseek", rawFD, -2, SEEK_END, interruptable: false) { + _ in + _ = try fd.seek(offset: -2, from: .end) + }, + + MockTestCase(name: "write", rawFD, writeBufAddr, bufCount, interruptable: true) { + retryOnInterrupt in + _ = try fd.write(writeBuf, retryOnInterrupt: retryOnInterrupt) + }, + + MockTestCase(name: "pwrite", rawFD, writeBufAddr, bufCount, 7, interruptable: true) { + retryOnInterrupt in + _ = try fd.write(toAbsoluteOffset: 7, writeBuf, retryOnInterrupt: retryOnInterrupt) + }, + + MockTestCase(name: "close", rawFD, interruptable: false) { + _ in + _ = try fd.close() + }, + + ] + + for test in syscallTestCases { test.runAllTests() } + } + + func testHelpers() { + // TODO: Test writeAll, writeAll(toAbsoluteOffset), closeAfter + } + + func testAdHocOpen() { + // Ad-hoc test touching a file system. + do { + // TODO: Test this against a virtual in-memory file system + let fd = try FileDescriptor.open("/tmp/b.txt", .readWrite, options: [.create, .truncate], permissions: .ownerReadWrite) + try fd.closeAfter { + try fd.writeAll("abc".utf8) + var def = "def" + try def.withUTF8 { + _ = try fd.write(UnsafeRawBufferPointer($0)) + } + try fd.seek(offset: 1, from: .start) + + let readLen = 3 + let readBytes = try Array(unsafeUninitializedCapacity: readLen) { (buf, count) in + count = try fd.read(into: UnsafeMutableRawBufferPointer(buf)) + } + let preadBytes = try Array(unsafeUninitializedCapacity: readLen) { (buf, count) in + count = try fd.read(fromAbsoluteOffset: 1, into: UnsafeMutableRawBufferPointer(buf)) + } + + XCTAssertEqual(readBytes.first!, "b".utf8.first!) + XCTAssertEqual(readBytes, preadBytes) + + // TODO: seek + } + } catch let err as Errno { + print("caught \(err))") + // Should we assert? I'd be interested in knowing if this happened + XCTAssert(false) + } catch { + fatalError() + } + } } diff --git a/Tests/SystemTests/TestingInfrastructure.swift b/Tests/SystemTests/TestingInfrastructure.swift new file mode 100644 index 00000000..03f93872 --- /dev/null +++ b/Tests/SystemTests/TestingInfrastructure.swift @@ -0,0 +1,132 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2020 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information +*/ + +import XCTest +import SystemInternals +import SystemPackage + +internal protocol TestCase { + // TODO: want a source location stack, more fidelity, kinds of stack entries, etc + var file: StaticString { get } + var line: UInt { get } + + // TODO: Instead have an attribute to register a test in a allTests var, similar to the argument parser. + func runAllTests() +} +extension TestCase { + func expectEqualSequence( + _ actual: S1, _ expected: S2, + _ message: String? = nil + ) where S1.Element: Equatable, S1.Element == S2.Element { + if !actual.elementsEqual(expected) { + fail(message) + } + } + func expectEqual( + _ actual: E, _ expected: E, + _ message: String? = nil + ) { + if actual != expected { + fail(message) + } + } + func expectTrue( + _ actual: Bool, + _ message: String? = nil + ) { + expectEqual(true, actual, message) + } + func expectFalse( + _ actual: Bool, + _ message: String? = nil + ) { + expectEqual(false, actual, message) + } + + func fail(_ reason: String? = nil) { + XCTAssert(false, reason ?? "", file: file, line: line) + } +} + +internal struct MockTestCase: TestCase { + var file: StaticString + var line: UInt + + var expected: Trace.Entry + var interruptable: Bool + + var body: (_ retryOnInterrupt: Bool) throws -> () + + init( + _ file: StaticString = #file, + _ line: UInt = #line, + name: String, + _ args: AnyHashable..., + interruptable: Bool, + _ body: @escaping (_ retryOnInterrupt: Bool) throws -> () + ) { + self.file = file + self.line = line + self.expected = Trace.Entry(name: name, args) + self.interruptable = interruptable + self.body = body + } + + func runAllTests() { + MockingDriver.withMockingEnabled { mocking in + // Make sure we completely match the trace queue + self.expectTrue(mocking.trace.isEmpty) + defer { self.expectTrue(mocking.trace.isEmpty) } + + // Test our API mappings to the lower-level syscall invocation + do { + try body(true) + self.expectEqual(mocking.trace.dequeue(), self.expected) + } catch { + self.fail() + } + + // Test interupt behavior. Interruptable calls will be told not to + // retry to catch the EINTR. Non-interruptable calls will be told to + // retry, to make sure they don't spin (e.g. if API changes to include + // interruptable) + do { + let oldErrno = mocking.forceErrno + mocking.forceErrno = .always(errno: EINTR) + defer { mocking.forceErrno = oldErrno } + try body(!interruptable) + self.fail() + } catch Errno.interrupted { + // Success! + self.expectEqual(mocking.trace.dequeue(), self.expected) + } catch { + self.fail() + } + + // Force a limited number of EINTRs, and make sure interruptable functions + // retry that number of times. Non-interruptable functions should throw it. + do { + let origForceErrno = mocking.forceErrno + defer { mocking.forceErrno = origForceErrno } + mocking.forceErrno = .counted(errno: EINTR, count: 3) + + try body(interruptable) + self.expectEqual(mocking.trace.dequeue(), self.expected) // EINTR + self.expectEqual(mocking.trace.dequeue(), self.expected) // EINTR + self.expectEqual(mocking.trace.dequeue(), self.expected) // EINTR + self.expectEqual(mocking.trace.dequeue(), self.expected) // Success + } catch Errno.interrupted { + self.expectFalse(interruptable) + self.expectEqual(mocking.trace.dequeue(), self.expected) // EINTR + } catch { + self.fail() + } + } + } +}