Add a hooks_testing library (#1952)

Previous test for methods like `expect`, `expectLater`, and
`expectAsync` used internal details of the test runner like `LiveTest`
to check behaviors such as holding the test open and specific failure
behavior.

Add an abstraction `TestCaseMonitor` to hide the implementation details
of creating and running a `LocalTest`. Expose only the parts of
`LiveTest` which are used by current tests - the state and errors.
The new class is defined in a new library `hooks_testing` since it is
intended primarily for writing tests for code that uses the `hooks`
library.
Add a new `State` enum to hide some details of `LiveTest`.
There is already a class named `State` that holds the `Status` and
`Result` for a `LiveTest`. This `State` attempts to encode the useful
parts of the status and result into a single enum. We don't get much use
out of the separate status and result. The `Result` is always `passed`
when the `Status` is anything other than `complete` (it shouldn't be
read), and changing the `Result` to anything else is _always_
accompanied by setting the `status` to `complete`. The new enum also
lets us hide the `failure` and `error` distinction instead of adding new
dependencies to it.
This works towards #1465

Migrate tests under `test_api/test/frontend` for the tests which will be
migrating to `package:matcher` to use the new APIs. Move to a
`utils_new.dart` file which will be moved along with these tests, and
the old `utils.dart` file will remain for testing the scaffolding APIs.
Do not replace the `expectTestsBlock` utility, the usage of this utility
is not any more clear or readable than spelling out the full behavior in
the test.

Add tests in `checks` for the behavior of holding the test for pending
work, and for unawaited failures.
Add utilities for testing against a `TestMonitor` locally within the
test.

pkgs/checks/pubspec.yaml

@@ -11,7 +11,7 @@ environment:
 dependencies:
   async: ^2.8.0
   meta: ^1.9.0
-  test_api: ^0.4.0
+  test_api: ^0.5.0
 
 dev_dependencies:
   test: ^1.21.3

pkgs/checks/pubspec_overrides.yaml

@@ -0,0 +1,7 @@
+dependency_overrides:
+  test_api:
+    path: ../test_api
+  test_core:
+    path: ../test_core
+  test:
+    path: ../test

pkgs/checks/test/context_test.dart

@@ -0,0 +1,176 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+
+import 'package:async/async.dart' hide Result;
+import 'package:checks/checks.dart';
+import 'package:checks/context.dart';
+import 'package:test/scaffolding.dart';
+import 'package:test_api/hooks.dart';
+import 'package:test_api/hooks_testing.dart';
+
+import 'test_shared.dart';
+
+void main() {
+  group('Context', () {
+    test('expectAsync holds test open', () async {
+      late void Function() callback;
+      final monitor = TestCaseMonitor.start(() {
+        check(null).context.expectAsync(() => [''], (actual) async {
+          final completer = Completer<void>();
+          callback = completer.complete;
+          await completer.future;
+          return null;
+        });
+      });
+      await pumpEventQueue();
+      check(monitor).state.equals(State.running);
+      callback();
+      await monitor.onDone;
+      check(monitor).didPass();
+    });
+
+    test('expectAsync does not hold test open past exception', () async {
+      late void Function() callback;
+      final monitor = TestCaseMonitor.start(() {
+        check(null).context.expectAsync(() => [''], (actual) async {
+          final completer = Completer<void>();
+          callback = completer.complete;
+          await completer.future;
+          throw 'oh no!';
+        });
+      });
+      await pumpEventQueue();
+      check(monitor).state.equals(State.running);
+      callback();
+      await monitor.onDone;
+      check(monitor)
+        ..state.equals(State.failed)
+        ..errors.single.has((e) => e.error, 'error').equals('oh no!');
+    });
+
+    test('nestAsync holds test open', () async {
+      late void Function() callback;
+      final monitor = TestCaseMonitor.start(() {
+        check(null).context.nestAsync(() => [''], (actual) async {
+          final completer = Completer<void>();
+          callback = completer.complete;
+          await completer.future;
+          return Extracted.value(null);
+        }, null);
+      });
+      await pumpEventQueue();
+      check(monitor).state.equals(State.running);
+      callback();
+      await monitor.onDone;
+      check(monitor).didPass();
+    });
+
+    test('nestAsync holds test open past async condition', () async {
+      late void Function() callback;
+      final monitor = TestCaseMonitor.start(() {
+        check(null).context.nestAsync(() => [''], (actual) async {
+          return Extracted.value(null);
+        }, LazyCondition((it) async {
+          final completer = Completer<void>();
+          callback = completer.complete;
+          await completer.future;
+        }));
+      });
+      await pumpEventQueue();
+      check(monitor).state.equals(State.running);
+      callback();
+      await monitor.onDone;
+      check(monitor).didPass();
+    });
+
+    test('nestAsync does not hold test open past exception', () async {
+      late void Function() callback;
+      final monitor = TestCaseMonitor.start(() {
+        check(null).context.nestAsync(() => [''], (actual) async {
+          final completer = Completer<void>();
+          callback = completer.complete;
+          await completer.future;
+          throw 'oh no!';
+        }, null);
+      });
+      await pumpEventQueue();
+      check(monitor).state.equals(State.running);
+      callback();
+      await monitor.onDone;
+      check(monitor)
+        ..state.equals(State.failed)
+        ..errors.single.has((e) => e.error, 'error').equals('oh no!');
+    });
+
+    test('expectUnawaited can fail the test after it completes', () async {
+      late void Function() callback;
+      final monitor = await TestCaseMonitor.run(() {
+        check(null).context.expectUnawaited(() => [''], (actual, reject) {
+          final completer = Completer<void>()
+            ..future.then((_) {
+              reject(Rejection(which: ['foo']));
+            });
+          callback = completer.complete;
+        });
+      });
+      check(monitor).state.equals(State.passed);
+      callback();
+      await pumpEventQueue();
+      check(monitor)
+        ..state.equals(State.failed)
+        ..errors.unorderedMatches([
+          it()
+            ..isA<AsyncError>()
+                .has((e) => e.error, 'error')
+                .isA<TestFailure>()
+                .has((f) => f.message, 'message')
+                .isNotNull()
+                .endsWith('Which: foo'),
+          it()
+            ..isA<AsyncError>()
+                .has((e) => e.error, 'error')
+                .isA<String>()
+                .startsWith('This test failed after it had already completed.')
+        ]);
+    });
+  });
+
+  group('SkipExtension', () {
+    test('marks the test as skipped', () async {
+      final monitor = await TestCaseMonitor.run(() {
+        check(null).skip('skip').isNotNull();
+      });
+      check(monitor).state.equals(State.skipped);
+    });
+  });
+}
+
+extension _MonitorChecks on Subject<TestCaseMonitor> {
+  Subject<State> get state => has((m) => m.state, 'state');
+  Subject<Iterable<AsyncError>> get errors => has((m) => m.errors, 'errors');
+  Subject<StreamQueue<AsyncError>> get onError =>
+      has((m) => m.onError, 'onError').withQueue;
+
+  /// Expects that the monitored test is completed as success with no errors.
+  ///
+  /// Sets up an unawaited expectation that the test does not emit errors in the
+  /// future in addition to checking there have been no errors yet.
+  void didPass() {
+    errors.isEmpty();
+    state.equals(State.passed);
+    onError.context.expectUnawaited(() => ['emits no further errors'],
+        (actual, reject) async {
+      await for (var error in actual.rest) {
+        reject(Rejection(which: [
+          ...prefixFirst('threw late error', literal(error.error)),
+          ...(const LineSplitter().convert(
+              TestHandle.current.formatStackTrace(error.stackTrace).toString()))
+        ]));
+      }
+    });
+  }
+}

pkgs/checks/test/test_shared.dart

@@ -59,7 +59,7 @@ extension RejectionChecks<T> on Subject<T> {
         ]);
       }
       return Extracted.value(failure.rejection);
-    }, _LazyCondition((rejection) {
+    }, LazyCondition((rejection) {
       if (didRunCallback) {
         rejection
             .has((r) => r.actual, 'actual')
@@ -96,9 +96,9 @@ extension ConditionChecks<T> on Subject<Condition<T>> {
 ///
 /// Allows basing the following condition in `isRejectedByAsync` on the actual
 /// value.
-class _LazyCondition<T> implements Condition<T> {
+class LazyCondition<T> implements Condition<T> {
   final FutureOr<void> Function(Subject<T>) _apply;
-  _LazyCondition(this._apply);
+  LazyCondition(this._apply);
 
   @override
   void apply(Subject<T> subject) {

pkgs/test_api/CHANGELOG.md

@@ -6,6 +6,8 @@
   major release.
 * **BREAKING** Add required `defaultCompiler` and `supportedCompilers` fields
   to `Runtime`.
+* Add `package:test_api/hooks_testing.dart` library for writing tests against
+  code that uses `package:test_api/hooks.dart`.
 
 ## 0.4.18
 

pkgs/test_api/lib/hooks_testing.dart

@@ -0,0 +1,156 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import 'src/backend/group.dart';
+import 'src/backend/invoker.dart';
+import 'src/backend/live_test.dart';
+import 'src/backend/metadata.dart';
+import 'src/backend/runtime.dart';
+import 'src/backend/state.dart';
+import 'src/backend/suite.dart';
+import 'src/backend/suite_platform.dart';
+
+export 'src/backend/state.dart' show Result, Status;
+export 'src/backend/test_failure.dart' show TestFailure;
+
+/// A monitor for the behavior of a callback when it is run as the body of a
+/// test case.
+///
+/// Allows running a callback as the body of a local test case and querying for
+/// the current [state], and [errors], and subscribing to future errors.
+///
+/// Use [run] to run a test body and query for the success or failure.
+///
+/// Use [start] to start a test and query for whether it has finished running.
+class TestCaseMonitor {
+  final LiveTest _liveTest;
+  final _done = Completer<void>();
+  TestCaseMonitor._(FutureOr<void> Function() body)
+      : _liveTest = _createTest(body);
+
+  /// Run [body] as a test case and return a [TestCaseMonitor] with the result.
+  ///
+  /// The [state] will either [State.passed], [State.skipped], or
+  /// [State.failed], the test will no longer be running.
+  ///
+  /// {@template result-late-fail}
+  /// Note that a test can change state from [State.passed] to [State.failed]
+  /// if the test surfaces an unawaited asynchronous error.
+  /// {@endtemplate}
+  ///
+  /// ```dart
+  /// final monitor = await TestCaseMonitor.run(() {
+  ///   fail('oh no!');
+  /// });
+  /// assert(monitor.state == State.failed);
+  /// assert((monitor.errors.single.error as TestFailure).message == 'oh no!');
+  /// ```
+  static Future<TestCaseMonitor> run(FutureOr<void> Function() body) async {
+    final monitor = TestCaseMonitor.start(body);
+    await monitor.onDone;
+    return monitor;
+  }
+
+  /// Start [body] as a test case and return a [TestCaseMonitor] with the status
+  /// and result.
+  ///
+  /// The [state] will start as [State.pending] if queried synchronously, but it
+  /// will switch to [State.running]. After `onDone` completes the state will be
+  /// one of [State.passed], [State.skipped], or [State.failed].
+  ///
+  /// {@macro result-late-fail}
+  ///
+  /// ```dart
+  /// late void Function() completeWork;
+  /// final monitor = TestCaseMonitor.start(() {
+  ///   final outstandingWork = TestHandle.current.markPending();
+  ///   completeWork = outstandingWork.complete;
+  /// });
+  /// await pumpEventQueue();
+  /// assert(monitor.state == State.running);
+  /// completeWork();
+  /// await monitor.onDone;
+  /// assert(monitor.state == State.passed);
+  /// ```
+  static TestCaseMonitor start(FutureOr<void> Function() body) =>
+      TestCaseMonitor._(body).._start();
+
+  void _start() {
+    _liveTest.run().whenComplete(_done.complete);
+  }
+
+  /// A future that completes after this test has finished running, or has
+  /// surfaced an error.
+  Future<void> get onDone => _done.future;
+
+  /// The running and success or failure status for the test case.
+  State get state {
+    final status = _liveTest.state.status;
+    if (status == Status.pending) return State.pending;
+    if (status == Status.running) return State.running;
+    final result = _liveTest.state.result;
+    if (result == Result.skipped) return State.skipped;
+    if (result == Result.success) return State.passed;
+    // result == Result.failure || result == Result.error
+    return State.failed;
+  }
+
+  /// The errors surfaced by the test.
+  ///
+  /// A test with any errors will have a [state] of [State.failed].
+  ///
+  /// {@macro result-late-fail}
+  ///
+  /// A test may have more than one error if there were unhandled asynchronous
+  /// errors surfaced after the test is done.
+  Iterable<AsyncError> get errors => _liveTest.errors;
+
+  /// A stream of errors surfaced by the test.
+  ///
+  /// This stream will not close, asynchronous errors may be surfaced within the
+  /// test's error zone at any point.
+  Stream<AsyncError> get onError => _liveTest.onError;
+}
+
+/// Returns a local [LiveTest] that runs [body].
+LiveTest _createTest(FutureOr<void> Function() body) {
+  var test = LocalTest('test', Metadata(chainStackTraces: true), body);
+  var suite = Suite(Group.root([test]), _suitePlatform, ignoreTimeouts: false);
+  return test.load(suite);
+}
+
+/// A dummy suite platform to use for testing suites.
+final _suitePlatform =
+    SuitePlatform(Runtime.vm, compiler: Runtime.vm.defaultCompiler);
+
+/// The running and success state of a test monitored by a [TestCaseMonitor].
+enum State {
+  /// The test is has not yet started.
+  pending,
+
+  /// The test is running and has not yet failed.
+  running,
+
+  /// The test has completed without any error.
+  ///
+  /// This implies that the test body has completed, and no error has surfaced
+  /// *yet*. However, it this doesn't mean that the test won't fail in the
+  /// future.
+  passed,
+
+  /// The test, or some part of it, has been skipped.
+  ///
+  /// This does not imply that the test has not had an error, but if there are
+  /// errors they are ignored.
+  skipped,
+
+  /// The test has failed.
+  ///
+  /// An test fails when any exception, typically a [TestFailure], is thrown in
+  /// the test's zone. A test that has failed may still have additional errors
+  /// that surface as unhandled asynchronous errors.
+  failed,
+}