Skip to content

Commit

Permalink
Lazily evaluate test arguments only after determining their test will…
Browse files Browse the repository at this point in the history 
… run, and support throwing expressions

Resolves #166
Resolves rdar://121531170
  • Loading branch information
@stmontgomery
stmontgomery committed Apr 24, 2024
1 parent 6676949 commit c4387af
Show file tree
Hide file tree
Showing 9 changed files with 334 additions and 88 deletions.
26 changes: 23 additions & 3 deletions Sources/Testing/Running/Runner.Plan.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,11 +193,14 @@ extension Runner.Plan {
testGraph = configuration.testFilter.apply(to: testGraph)

// For each test value, determine the appropriate action for it.
await testGraph.forEach { keyPath, test in
//
// FIXME: Parallelize this work. Calling `prepare(...)` on all traits and
// evaluating all test arguments should be safely parallelizable.
testGraph = await testGraph.mapValues { keyPath, test in
// Skip any nil test, which implies this node is just a placeholder and
// not actual test content.
guard let test else {
return
guard var test else {
return nil
}

var action = runAction
Expand Down Expand Up @@ -232,7 +235,24 @@ extension Runner.Plan {
action = .recordIssue(issue)
}

// If the test is still planned to run (i.e. nothing thus far has caused
// it to be skipped), evaluate its test cases now.
//
// The argument expressions of each test are captured in closures so they
// can be evaluated lazily only once it is determined that the test will
// run, to avoid unnecessary work. But now is the appropriate time to
// evaluate them.
do {
try await test.evaluateTestCases()
} catch {
let sourceContext = SourceContext(backtrace: Backtrace(forFirstThrowOf: error))
let issue = Issue(kind: .errorCaught(error), comments: [], sourceContext: sourceContext)
action = .recordIssue(issue)
}

actionGraph.updateValue(action, at: keyPath)

return test
}

// Now that we have allowed all the traits to update their corresponding
Expand Down
26 changes: 14 additions & 12 deletions Sources/Testing/Test+Macro.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ extension Test {
testFunction: @escaping @Sendable () async throws -> Void
) -> Self {
let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
let caseGenerator = Case.Generator(testFunction: testFunction)
let caseGenerator = { @Sendable in Case.Generator(testFunction: testFunction) }
return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: [])
}
}
Expand Down Expand Up @@ -236,14 +236,14 @@ extension Test {
xcTestCompatibleSelector: __XCTestCompatibleSelector?,
displayName: String? = nil,
traits: [any TestTrait],
arguments collection: C,
arguments collection: @escaping @Sendable () async throws -> C,
sourceLocation: SourceLocation,
parameters paramTuples: [__Parameter],
testFunction: @escaping @Sendable (C.Element) async throws -> Void
) -> Self where C: Collection & Sendable, C.Element: Sendable {
let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
let parameters = paramTuples.parameters
let caseGenerator = Case.Generator(arguments: collection, parameters: parameters, testFunction: testFunction)
let caseGenerator = { @Sendable in Case.Generator(arguments: try await collection(), parameters: parameters, testFunction: testFunction) }
return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
}
}
Expand Down Expand Up @@ -365,14 +365,14 @@ extension Test {
xcTestCompatibleSelector: __XCTestCompatibleSelector?,
displayName: String? = nil,
traits: [any TestTrait],
arguments collection1: C1, _ collection2: C2,
arguments collection1: @escaping @Sendable () async throws -> C1, _ collection2: @escaping @Sendable () async throws -> C2,
sourceLocation: SourceLocation,
parameters paramTuples: [__Parameter],
testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void
) -> Self where C1: Collection & Sendable, C1.Element: Sendable, C2: Collection & Sendable, C2.Element: Sendable {
let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
let parameters = paramTuples.parameters
let caseGenerator = Case.Generator(arguments: collection1, collection2, parameters: parameters, testFunction: testFunction)
let caseGenerator = { @Sendable in try await Case.Generator(arguments: collection1(), collection2(), parameters: parameters, testFunction: testFunction) }
return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
}

Expand All @@ -389,14 +389,14 @@ extension Test {
xcTestCompatibleSelector: __XCTestCompatibleSelector?,
displayName: String? = nil,
traits: [any TestTrait],
arguments collection: C,
arguments collection: @escaping @Sendable () async throws -> C,
sourceLocation: SourceLocation,
parameters paramTuples: [__Parameter],
testFunction: @escaping @Sendable ((E1, E2)) async throws -> Void
) -> Self where C: Collection & Sendable, C.Element == (E1, E2), E1: Sendable, E2: Sendable {
let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
let parameters = paramTuples.parameters
let caseGenerator = Case.Generator(arguments: collection, parameters: parameters, testFunction: testFunction)
let caseGenerator = { @Sendable in Case.Generator(arguments: try await collection(), parameters: parameters, testFunction: testFunction) }
return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
}

Expand All @@ -416,14 +416,14 @@ extension Test {
xcTestCompatibleSelector: __XCTestCompatibleSelector?,
displayName: String? = nil,
traits: [any TestTrait],
arguments dictionary: Dictionary<Key, Value>,
arguments dictionary: @escaping @Sendable () async throws -> Dictionary<Key, Value>,
sourceLocation: SourceLocation,
parameters paramTuples: [__Parameter],
testFunction: @escaping @Sendable ((Key, Value)) async throws -> Void
) -> Self where Key: Sendable, Value: Sendable {
let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
let parameters = paramTuples.parameters
let caseGenerator = Case.Generator(arguments: dictionary, parameters: parameters, testFunction: testFunction)
let caseGenerator = { @Sendable in Case.Generator(arguments: try await dictionary(), parameters: parameters, testFunction: testFunction) }
return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
}

Expand All @@ -437,15 +437,17 @@ extension Test {
xcTestCompatibleSelector: __XCTestCompatibleSelector?,
displayName: String? = nil,
traits: [any TestTrait],
arguments zippedCollections: Zip2Sequence<C1, C2>,
arguments zippedCollections: @escaping @Sendable () async throws -> Zip2Sequence<C1, C2>,
sourceLocation: SourceLocation,
parameters paramTuples: [__Parameter],
testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void
) -> Self where C1: Collection & Sendable, C1.Element: Sendable, C2: Collection & Sendable, C2.Element: Sendable {
let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
let parameters = paramTuples.parameters
let caseGenerator = Case.Generator(arguments: zippedCollections, parameters: parameters) {
try await testFunction($0, $1)
let caseGenerator = { @Sendable in
Case.Generator(arguments: try await zippedCollections(), parameters: parameters) {
try await testFunction($0, $1)
}
}
return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
}
Expand Down

0 comments on commit c4387af

Please sign in to comment.