From 8822967ae18dad92af16534326b99ce7fa50feb6 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Mon, 6 Feb 2023 14:54:25 -0800 Subject: [PATCH 1/5] Finesse host application warning Just a lil cleanup work from #48. --- Sources/XCTestDynamicOverlay/XCTFail.swift | 155 +++++++++--------- .../HostAppDetectionTests.swift | 54 ++---- 2 files changed, 91 insertions(+), 118 deletions(-) diff --git a/Sources/XCTestDynamicOverlay/XCTFail.swift b/Sources/XCTestDynamicOverlay/XCTFail.swift index 22f96ff6..a82ffb1a 100644 --- a/Sources/XCTestDynamicOverlay/XCTFail.swift +++ b/Sources/XCTestDynamicOverlay/XCTFail.swift @@ -1,6 +1,6 @@ -#if DEBUG - import Foundation +import Foundation +#if DEBUG #if canImport(ObjectiveC) /// This function generates a failure immediately and unconditionally. /// @@ -12,7 +12,8 @@ /// results. @_disfavoredOverload public func XCTFail(_ message: String = "") { - let message = appendHostAppWarningIfNeeded(message) + var message = message + attachHostApplicationWarningIfNeeded(&message) guard let currentTestCase = XCTCurrentTestCase, let XCTIssue = NSClassFromString("XCTIssue") @@ -46,7 +47,8 @@ /// results. @_disfavoredOverload public func XCTFail(_ message: String = "", file: StaticString, line: UInt) { - let message = appendHostAppWarningIfNeeded(message) + var message = message + attachHostApplicationWarningIfNeeded(&message) _XCTFailureHandler(nil, true, "\(file)", line, "\(message.isEmpty ? "failed" : message)", nil) } @@ -57,6 +59,74 @@ dlsym(dlopen(nil, RTLD_LAZY), "_XCTFailureHandler"), to: XCTFailureHandler.self ) + + private func attachHostApplicationWarningIfNeeded(_ message: inout String) { + guard + _XCTIsTesting, + Bundle.main.bundleIdentifier != "com.apple.dt.xctest.tool" + else { return } + + let callStack = Thread.callStackSymbols + + // Detect when synchronous test exists in stack. + guard callStack.allSatisfy({ frame in !frame.contains(" XCTestCore ") }) + else { return } + + // Detect when asynchronous test exists in stack. + guard callStack.allSatisfy({ frame in !isTestFrame(frame) }) + else { return } + + let displayName = + Bundle.main.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String + ?? Bundle.main.object(forInfoDictionaryKey: kCFBundleNameKey as String) as? String + ?? "Unknown host application" + + let bundleIdentifier = Bundle.main.bundleIdentifier ?? "Unknown bundle identifier" + + message.append(""" + + + ┏━━━━━━━━━━━━━━━━━┉┅ + ┃ ⚠︎ Warning: + ┃ + ┃ This failure was emitted from a host application outside the test stack. + ┃ + ┃ Host application: + ┃ \(displayName) (\(bundleIdentifier)) + ┃ + ┃ The host application may have emitted this failure when it first launched, + ┃ outside of the current test that happened to be running. + ┃ + ┃ Consider setting the test target's host application to "None," or prevent + ┃ the host application from performing the code path that emits failure. + ┗━━━━━━━━━━━━━━━━━┉┅ + + For more information (and workarounds), see "Testing gotchas": + + https://pointfreeco.github.io/swift-dependencies/main/documentation/dependencies/testing#testing-gotchas + """ + ) + } + + func isTestFrame(_ frame: String) -> Bool { + // Regular expression to detect and demangle an XCTest case frame: + // + // 1. `(?<=\$s)`: Starts with "$s" (stable mangling) + // 2. `\d{1,3}`: Some numbers (the class name length or the module name length) + // 3. `.*`: The class name, or module name + class name length + class name + // 4. `C`: The class type identifier + // 5. `(?=\d{1,3}test.*yy(Ya)?K?F)`: The function name length, a function that starts with + // `test`, has no arguments (`y`), returns Void (`y`), and is a function (`F`), potentially + // async (`Ya`), throwing (`K`), or both. + let mangledTestFrame = #"(?<=\$s)\d{1,3}.*C(?=\d{1,3}test.*yy(Ya)?K?F)"# + + guard let XCTestCase = NSClassFromString("XCTestCase") + else { return false } + + return frame.range(of: mangledTestFrame, options: .regularExpression) + .map { (_typeByName(String(frame[$0])) as? NSObject.Type)?.isSubclass(of: XCTestCase) ?? false } + ?? false + } #elseif canImport(XCTest) // NB: It seems to be safe to import XCTest on Linux @_exported import func XCTest.XCTFail @@ -66,83 +136,6 @@ @_disfavoredOverload public func XCTFail(_ message: String = "", file: StaticString, line: UInt) {} #endif - - func appendHostAppWarningIfNeeded(_ originalMessage: String) -> String { - guard _XCTIsTesting else { return originalMessage } - if Bundle.main.bundleIdentifier == "com.apple.dt.xctest.tool" // Apple platforms - || Bundle.main.bundleIdentifier == nil // Linux - { - // XCTesting is providing a default host app. - return originalMessage - } - - if Thread.callStackSymbols.contains(where: { $0.range(of: "XCTestCore") != nil }) { - // We are apparently performing a sync test - return originalMessage - } - - if testCaseSubclass(callStackSymbols: Thread.callStackSymbols) != nil { - // We are apparently performing an async test. - // We're matching a `() -> ()` function that starts with `test`, from a `XCTestCase` subclass - return originalMessage - } - - let message = """ - Warning! This failure occurred while running tests hosted by the main app. - - Testing using the main app as a host can lead to false positive test failures created by the \ - app accessing unimplemented values itself when it is spun up. - - - Test host: \(Bundle.main.bundleIdentifier ?? "Unknown") - - You can find more information and workarounds in the "Testing/Testing Gotchas" section of \ - Dependencies' documentation at \ - https://pointfreeco.github.io/swift-dependencies/main/documentation/dependencies/testing/. - """ - - return [originalMessage, "", message].joined(separator: "\n") - } - - // (?<=\$s): Starts with "$s" (stable mangling); - // \d{1,3}: Some numbers (the class name length or the module name length); - // .*: The class name, or module name + class name length + class name; - // C: The class type identifier; - // (?=\d{1,3}test.*yy(Ya)?K?F): Followed by the function name length, function that starts with - // `test`, has no arguments (y), returns Void (y), and is a function (F), potentially async (Ya), - // throwing (K), or both. - private let testCaseRegex = #"(?<=\$s)\d{1,3}.*C(?=\d{1,3}test.*yy(Ya)?K?F)"# - - func testCaseSubclass(callStackSymbols: [String]) -> Any.Type? { - for frame in callStackSymbols { - var startIndex = frame.startIndex - while startIndex != frame.endIndex { - if let range = frame.range( - of: testCaseRegex, - options: .regularExpression, - range: startIndex.. Any.Type? { - if let object = _typeByName(mangledName) as? NSObject.Type, - NSClassFromString("XCTestCase").map(object.isSubclass(of:)) == true - { - return object - } - return nil - } - #else /// This function generates a failure immediately and unconditionally. /// diff --git a/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift b/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift index 95b029ec..44c13adb 100644 --- a/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift +++ b/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift @@ -1,43 +1,23 @@ -import XCTest +#if DEBUG + import XCTest -@testable import XCTestDynamicOverlay + @testable import XCTestDynamicOverlay -final class HostAppCallStackTests: XCTestCase { - func testIsAbleToDetectTest() { - XCTAssertEqual( - testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init), - ObjectIdentifier(HostAppCallStackTests.self) - ) - } - - func testIsAbleToDetectAsyncTest() async { - XCTAssertEqual( - testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init), - ObjectIdentifier(HostAppCallStackTests.self) - ) - } + final class HostAppCallStackTests: XCTestCase { + func testIsAbleToDetectTest() { + XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame)) + } - func testIsAbleToDetectThrowingTest() throws { - XCTAssertEqual( - testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init), - ObjectIdentifier(HostAppCallStackTests.self) - ) - } + func testIsAbleToDetectAsyncTest() async { + XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame)) + } - func testIsAbleToDetectAsyncThrowingTest() async throws { - XCTAssertEqual( - testCaseSubclass(callStackSymbols: Thread.callStackSymbols).map(ObjectIdentifier.init), - ObjectIdentifier(HostAppCallStackTests.self) - ) - } + func testIsAbleToDetectThrowingTest() throws { + XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame)) + } - #if !os(Linux) - func testFailDoesNotAppendHostAppWarningFromATest() { - XCTExpectFailure { - XCTestDynamicOverlay.XCTFail("foo") - } issueMatcher: { - $0.compactDescription == "foo" - } + func testIsAbleToDetectAsyncThrowingTest() async throws { + XCTAssert(Thread.callStackSymbols.contains(where: isTestFrame)) } - #endif -} + } +#endif From 0048aa20b9191a58dee8d6d669b4d842c6536ce1 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Mon, 6 Feb 2023 14:57:29 -0800 Subject: [PATCH 2/5] wip --- Sources/XCTestDynamicOverlay/XCTFail.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/XCTestDynamicOverlay/XCTFail.swift b/Sources/XCTestDynamicOverlay/XCTFail.swift index a82ffb1a..6c7331da 100644 --- a/Sources/XCTestDynamicOverlay/XCTFail.swift +++ b/Sources/XCTestDynamicOverlay/XCTFail.swift @@ -95,7 +95,7 @@ import Foundation ┃ \(displayName) (\(bundleIdentifier)) ┃ ┃ The host application may have emitted this failure when it first launched, - ┃ outside of the current test that happened to be running. + ┃ outside this current test that happens to be running. ┃ ┃ Consider setting the test target's host application to "None," or prevent ┃ the host application from performing the code path that emits failure. From 34bd4774458d8f62f77de02eccc78668ae60a54d Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Mon, 6 Feb 2023 14:58:32 -0800 Subject: [PATCH 3/5] wip --- Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift b/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift index 44c13adb..eb2de8d7 100644 --- a/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift +++ b/Tests/XCTestDynamicOverlayTests/HostAppDetectionTests.swift @@ -1,4 +1,4 @@ -#if DEBUG +#if DEBUG && canImport(ObjectiveC) import XCTest @testable import XCTestDynamicOverlay From 42375bc9e30d9de96c5c94aeb761b2ab68199402 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Mon, 6 Feb 2023 15:01:37 -0800 Subject: [PATCH 4/5] wip --- Sources/XCTestDynamicOverlay/XCTFail.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/XCTestDynamicOverlay/XCTFail.swift b/Sources/XCTestDynamicOverlay/XCTFail.swift index 6c7331da..643aa024 100644 --- a/Sources/XCTestDynamicOverlay/XCTFail.swift +++ b/Sources/XCTestDynamicOverlay/XCTFail.swift @@ -83,6 +83,10 @@ import Foundation let bundleIdentifier = Bundle.main.bundleIdentifier ?? "Unknown bundle identifier" + if !message.contains(where: \.isNewline) { + message.append(" …") + } + message.append(""" From 7568e3d17dbf9c019ea977175da1200c1d8527a8 Mon Sep 17 00:00:00 2001 From: Stephen Celis Date: Mon, 6 Feb 2023 15:49:08 -0800 Subject: [PATCH 5/5] wip --- Sources/XCTestDynamicOverlay/XCTFail.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/XCTestDynamicOverlay/XCTFail.swift b/Sources/XCTestDynamicOverlay/XCTFail.swift index 643aa024..90e65b67 100644 --- a/Sources/XCTestDynamicOverlay/XCTFail.swift +++ b/Sources/XCTestDynamicOverlay/XCTFail.swift @@ -104,6 +104,7 @@ import Foundation ┃ Consider setting the test target's host application to "None," or prevent ┃ the host application from performing the code path that emits failure. ┗━━━━━━━━━━━━━━━━━┉┅ + ▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄▀▄ For more information (and workarounds), see "Testing gotchas":