Lazily evaluate test arguments only after determining their test will run, and support throwing expressions (#366)

This changes test declaration, discovery, planning, and running such
that arguments to parameterized test functions are evaluated lazily,
only after determining their test will run. It also adds support for
throwing expressions in test arguments.

### Motivation:

When defining parameterized test functions, it can be useful to call
throwing or asynchronous APIs — for example, to fetch the arguments from
an external source. Today, the `@Test` macro supports `async`
expressions but not `throws`. More problematic still: the arguments of
_all_ tests get evaluated, even for tests which don't actually run
(which may happen for various reasons). Evaluating the arguments of a
test which won't run wastes time and resources, so we should try to
avoid this, and support `throws` expressions too for better flexibility.

### Modifications:

- Modify the `@Test` macro and supporting library interfaces to surround
test arguments in closures, so their evaluation can be lazy and
deferred.
- Modify how test case arguments are stored on each `Test` instance
accordingly.
- Modify the planning logic in `Runner.Plan` to defer evaluation of test
cases until we determine whether each test will run.
- Update tests.

### Result:

Now, arguments to a parameterized test will only be evaluated if that
test is actually going to run, and the expressions may include `try`
and potentially throw an `Error`.

### Checklist:

- [x] Code and documentation should follow the style of the [Style
Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md).
- [x] If public symbols are renamed or modified, DocC references should
be updated.

Resolves #166
Resolves rdar://121531170

Author: stmontgomery

Sources/Testing/Running/Runner.Plan.swift

@@ -193,11 +193,14 @@ extension Runner.Plan {
     testGraph = configuration.testFilter.apply(to: testGraph)
 
     // For each test value, determine the appropriate action for it.
-    await testGraph.forEach { keyPath, test in
+    //
+    // FIXME: Parallelize this work. Calling `prepare(...)` on all traits and
+    // evaluating all test arguments should be safely parallelizable.
+    testGraph = await testGraph.mapValues { keyPath, test in
       // Skip any nil test, which implies this node is just a placeholder and
       // not actual test content.
-      guard let test else {
-        return
+      guard var test else {
+        return nil
       }
 
       var action = runAction
@@ -232,7 +235,24 @@ extension Runner.Plan {
         action = .recordIssue(issue)
       }
 
+      // If the test is still planned to run (i.e. nothing thus far has caused
+      // it to be skipped), evaluate its test cases now.
+      //
+      // The argument expressions of each test are captured in closures so they
+      // can be evaluated lazily only once it is determined that the test will
+      // run, to avoid unnecessary work. But now is the appropriate time to
+      // evaluate them.
+      do {
+        try await test.evaluateTestCases()
+      } catch {
+        let sourceContext = SourceContext(backtrace: Backtrace(forFirstThrowOf: error))
+        let issue = Issue(kind: .errorCaught(error), comments: [], sourceContext: sourceContext)
+        action = .recordIssue(issue)
+      }
+
       actionGraph.updateValue(action, at: keyPath)
+
+      return test
     }
 
     // Now that we have allowed all the traits to update their corresponding

Sources/Testing/Test+Macro.swift

@@ -168,7 +168,7 @@ extension Test {
     testFunction: @escaping @Sendable () async throws -> Void
   ) -> Self {
     let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
-    let caseGenerator = Case.Generator(testFunction: testFunction)
+    let caseGenerator = { @Sendable in Case.Generator(testFunction: testFunction) }
     return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: [])
   }
 }
@@ -239,14 +239,14 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments collection: C,
+    arguments collection: @escaping @Sendable () async throws -> C,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__Parameter],
     testFunction: @escaping @Sendable (C.Element) async throws -> Void
   ) -> Self where C: Collection & Sendable, C.Element: Sendable {
     let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
     let parameters = paramTuples.parameters
-    let caseGenerator = Case.Generator(arguments: collection, parameters: parameters, testFunction: testFunction)
+    let caseGenerator = { @Sendable in Case.Generator(arguments: try await collection(), parameters: parameters, testFunction: testFunction) }
     return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
   }
 }
@@ -370,14 +370,14 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments collection1: C1, _ collection2: C2,
+    arguments collection1: @escaping @Sendable () async throws -> C1, _ collection2: @escaping @Sendable () async throws -> C2,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__Parameter],
     testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void
   ) -> Self where C1: Collection & Sendable, C1.Element: Sendable, C2: Collection & Sendable, C2.Element: Sendable {
     let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
     let parameters = paramTuples.parameters
-    let caseGenerator = Case.Generator(arguments: collection1, collection2, parameters: parameters, testFunction: testFunction)
+    let caseGenerator = { @Sendable in try await Case.Generator(arguments: collection1(), collection2(), parameters: parameters, testFunction: testFunction) }
     return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
   }
 
@@ -394,14 +394,14 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments collection: C,
+    arguments collection: @escaping @Sendable () async throws -> C,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__Parameter],
     testFunction: @escaping @Sendable ((E1, E2)) async throws -> Void
   ) -> Self where C: Collection & Sendable, C.Element == (E1, E2), E1: Sendable, E2: Sendable {
     let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
     let parameters = paramTuples.parameters
-    let caseGenerator = Case.Generator(arguments: collection, parameters: parameters, testFunction: testFunction)
+    let caseGenerator = { @Sendable in Case.Generator(arguments: try await collection(), parameters: parameters, testFunction: testFunction) }
     return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
   }
 
@@ -421,14 +421,14 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments dictionary: Dictionary<Key, Value>,
+    arguments dictionary: @escaping @Sendable () async throws -> Dictionary<Key, Value>,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__Parameter],
     testFunction: @escaping @Sendable ((Key, Value)) async throws -> Void
   ) -> Self where Key: Sendable, Value: Sendable {
     let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
     let parameters = paramTuples.parameters
-    let caseGenerator = Case.Generator(arguments: dictionary, parameters: parameters, testFunction: testFunction)
+    let caseGenerator = { @Sendable in Case.Generator(arguments: try await dictionary(), parameters: parameters, testFunction: testFunction) }
     return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
   }
 
@@ -442,15 +442,17 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments zippedCollections: Zip2Sequence<C1, C2>,
+    arguments zippedCollections: @escaping @Sendable () async throws -> Zip2Sequence<C1, C2>,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__Parameter],
     testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void
   ) -> Self where C1: Collection & Sendable, C1.Element: Sendable, C2: Collection & Sendable, C2.Element: Sendable {
     let containingTypeInfo = containingType.map(TypeInfo.init(describing:))
     let parameters = paramTuples.parameters
-    let caseGenerator = Case.Generator(arguments: zippedCollections, parameters: parameters) {
-      try await testFunction($0, $1)
+    let caseGenerator = { @Sendable in
+      Case.Generator(arguments: try await zippedCollections(), parameters: parameters) {
+        try await testFunction($0, $1)
+      }
     }
     return Self(name: testFunctionName, displayName: displayName, traits: traits, sourceLocation: sourceLocation, containingTypeInfo: containingTypeInfo, xcTestCompatibleSelector: xcTestCompatibleSelector, testCases: caseGenerator, parameters: parameters)
   }

Sources/Testing/Test.swift

@@ -68,24 +68,91 @@ public struct Test: Sendable {
   @_spi(ForToolsIntegrationOnly)
   public var xcTestCompatibleSelector: __XCTestCompatibleSelector?
 
-  /// Storage for the ``testCases`` property.
+  /// An enumeration describing the evaluation state of a test's cases.
   ///
-  /// This use of `UncheckedSendable` and of `AnySequence` is necessary because
-  /// it is not currently possible to express `Sequence<Test.Case> & Sendable`
-  /// as an existential (`any`) ([96960993](rdar://96960993)). It is also not
-  /// possible to have a value of an underlying generic sequence type without
-  /// specifying its generic parameters.
-  private var _testCases: UncheckedSendable<AnySequence<Test.Case>>?
+  /// This use of `UncheckedSendable` and of `AnySequence` in this type's cases
+  /// is necessary because it is not currently possible to express
+  /// `Sequence<Test.Case> & Sendable` as an existential (`any`)
+  /// ([96960993](rdar://96960993)). It is also not possible to have a value of
+  /// an underlying generic sequence type without specifying its generic
+  /// parameters.
+  fileprivate enum TestCasesState: Sendable {
+    /// The test's cases have not yet been evaluated.
+    ///
+    /// - Parameters:
+    ///   - function: The function to call to evaluate the test's cases. The
+    ///     result is a sequence of test cases.
+    case unevaluated(_ function: @Sendable () async throws -> AnySequence<Test.Case>)
+
+    /// The test's cases have been evaluated, and either returned a sequence of
+    /// cases or failed by throwing an error.
+    ///
+    /// - Parameters:
+    ///   - result: The result of having evaluated the test's cases.
+    case evaluated(_ result: Result<UncheckedSendable<AnySequence<Test.Case>>, any Error>)
+  }
+
+  /// The evaluation state of this test's cases, if any.
+  ///
+  /// If this test represents a suite type, the value of this property is `nil`.
+  fileprivate var testCasesState: TestCasesState?
 
   /// The set of test cases associated with this test, if any.
   ///
+  /// - Precondition: This property may only be accessed on test instances
+  ///   representing suites, or on test functions whose ``testCaseState``
+  ///   indicates a successfully-evaluated state.
+  ///
   /// For parameterized tests, each test case is associated with a single
   /// combination of parameterized inputs. For non-parameterized tests, a single
   /// test case is synthesized. For test suite types (as opposed to test
   /// functions), the value of this property is `nil`.
-  @_spi(ForToolsIntegrationOnly)
-  public var testCases: (some Sequence<Test.Case> & Sendable)? {
-    _testCases?.rawValue
+  var testCases: (some Sequence<Test.Case> & Sendable)? {
+    guard let testCasesState else {
+      return nil as AnySequence<Test.Case>?
+    }
+    guard case let .evaluated(result) = testCasesState else {
+      // Callers are expected to first attempt to evaluate a test's cases by
+      // calling `evaluateTestCases()`.
+      preconditionFailure("Attempting to access test cases before they have been evaluated.")
+    }
+    guard case let .success(testCases) = result else {
+      // Callers are never expected to access this property after evaluating a
+      // test's cases, if that evaluation threw an error, because if the test
+      // cannot be run. In this scenario, a `Runner.Plan` is expected to record
+      // issue for the test, rather than attempt to run it, and thus never
+      // access this property.
+      preconditionFailure("Attempting to access test cases after evaluating them failed.")
+    }
+    return testCases.rawValue
+  }
+
+  /// Evaluate this test's cases if they have not been evaluated yet.
+  ///
+  /// The arguments of a test are captured into a closure so they can be lazily
+  /// evaluated only if the test will run to avoid unnecessary work. This
+  /// function may be called once that determination has been made, to perform
+  /// this evaluation once. The resulting arguments are stored on this instance
+  /// so that subsequent calls to ``testCases`` do not cause the arguments to be
+  /// re-evaluated.
+  ///
+  /// - Throws: Any error caught while first evaluating the test arguments.
+  mutating func evaluateTestCases() async throws {
+    guard let testCasesState else { return }
+
+    do {
+      switch testCasesState {
+      case let .unevaluated(function):
+        let sequence = try await function()
+        self.testCasesState = .evaluated(.success(UncheckedSendable(rawValue: sequence)))
+      case .evaluated:
+        // No-op: already evaluated
+        break
+      }
+    } catch {
+      self.testCasesState = .evaluated(.failure(error))
+      throw error
+    }
   }
 
   /// Whether or not this test is parameterized.
@@ -109,7 +176,7 @@ public struct Test: Sendable {
   ///
   /// A test suite can be declared using the ``Suite(_:_:)`` macro.
   public var isSuite: Bool {
-    containingTypeInfo != nil && testCases == nil
+    containingTypeInfo != nil && testCasesState == nil
   }
 
   /// Whether or not this instance was synthesized at runtime.
@@ -137,6 +204,27 @@ public struct Test: Sendable {
     self.isSynthesized = isSynthesized
   }
 
+  /// Initialize an instance of this type representing a test function.
+  init<S>(
+    name: String,
+    displayName: String? = nil,
+    traits: [any Trait],
+    sourceLocation: SourceLocation,
+    containingTypeInfo: TypeInfo? = nil,
+    xcTestCompatibleSelector: __XCTestCompatibleSelector? = nil,
+    testCases: @escaping @Sendable () async throws -> Test.Case.Generator<S>,
+    parameters: [Parameter]
+  ) {
+    self.name = name
+    self.displayName = displayName
+    self.traits = traits
+    self.sourceLocation = sourceLocation
+    self.containingTypeInfo = containingTypeInfo
+    self.xcTestCompatibleSelector = xcTestCompatibleSelector
+    self.testCasesState = .unevaluated { .init(try await testCases()) }
+    self.parameters = parameters
+  }
+
   /// Initialize an instance of this type representing a test function.
   init<S>(
     name: String,
@@ -154,7 +242,7 @@ public struct Test: Sendable {
     self.sourceLocation = sourceLocation
     self.containingTypeInfo = containingTypeInfo
     self.xcTestCompatibleSelector = xcTestCompatibleSelector
-    self._testCases = .init(rawValue: .init(testCases))
+    self.testCasesState = .evaluated(.success(UncheckedSendable(rawValue: .init(testCases))))
     self.parameters = parameters
   }
 }
@@ -209,10 +297,10 @@ extension Test {
 
     /// The set of test cases associated with this test, if any.
     ///
-    /// ## See Also
-    ///
-    /// - ``Test/testCases``
-    @_spi(ForToolsIntegrationOnly)
+    /// If the ``Test`` this instance was snapshotted from represented a
+    /// parameterized test function but its test cases had not yet been
+    /// evaluated when the snapshot was taken, or the evaluation attempt failed,
+    /// the value of this property will be an empty array.
     public var testCases: [Test.Case.Snapshot]?
 
     /// The test function parameters, if any.
@@ -252,14 +340,23 @@ extension Test {
       name = test.name
       displayName = test.displayName
       sourceLocation = test.sourceLocation
-      testCases = test.testCases?.map(Test.Case.Snapshot.init)
       parameters = test.parameters
       comments = test.comments
       tags = test.tags
       associatedBugs = test.associatedBugs
       if #available(_clockAPI, *) {
         _timeLimit = test.timeLimit.map(TimeValue.init)
       }
+
+      testCases = switch test.testCasesState {
+      case .unevaluated,
+           .evaluated(.failure):
+        []
+      case let .evaluated(.success(testCases)):
+        testCases.rawValue.map(Test.Case.Snapshot.init(snapshotting:))
+      case nil:
+        nil
+      }
     }
 
     /// Whether or not this test is parameterized.

Sources/TestingMacros/Support/AttributeDiscovery.swift

@@ -180,6 +180,21 @@ struct AttributeInfo {
         ArrayElementSyntax(expression: traitExpr)
       }
     }))
+
+    // Any arguments of the test declaration macro which specify test arguments
+    // need to be wrapped a closure so they may be evaluated lazily by the
+    // testing library at runtime. If any such arguments are present, they will
+    // begin with a labeled argument named `arguments:` and include all
+    // subsequent unlabeled arguments.
+    var otherArguments = self.otherArguments
+    if let argumentsIndex = otherArguments.firstIndex(where: { $0.label?.tokenKind == .identifier("arguments") }) {
+      for index in argumentsIndex ..< otherArguments.endIndex {
+        var argument = otherArguments[index]
+        argument.expression = .init(ClosureExprSyntax { argument.expression })
+        otherArguments[index] = argument
+      }
+    }
+
     arguments += otherArguments
     arguments.append(Argument(label: "sourceLocation", expression: sourceLocation))