Merge branch 'main' into origin/issues/3107-method-filters

build.gradle.kts

@@ -61,5 +61,5 @@ nexusPublishing {
 }
 
 nohttp {
-	source.exclude("**/.gradle/**", "gradle/plugins/**/build/**")
+	source.exclude("**/.gradle/**", "gradle/plugins/**/build/**", "buildSrc/build/**")
 }

documentation/documentation.gradle.kts

@@ -49,9 +49,6 @@ dependencies {
 	testRuntimeOnly(libs.apiguardian) {
 		because("it's required to generate API tables")
 	}
-	testRuntimeOnly(libs.openTestReporting.events) {
-		because("it's required to run tests via IntelliJ which does not consumed the shadowed jar of junit-platform-reporting")
-	}
 
 	testImplementation(libs.classgraph) {
 		because("ApiReportGenerator needs it")

documentation/src/docs/asciidoc/link-attributes.adoc

@@ -153,7 +153,7 @@ endif::[]
 :junit-jupiter-engine:                       {javadoc-root}/org.junit.jupiter.engine/org/junit/jupiter/engine/package-summary.html[junit-jupiter-engine]
 // Jupiter Extension Implementations
 :DisabledCondition:                          {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/DisabledCondition.java[DisabledCondition]
-:RepetitionInfoParameterResolver:            {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/RepetitionInfoParameterResolver.java[RepetitionInfoParameterResolver]
+:RepetitionExtension:                        {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/RepetitionExtension.java[RepetitionExtension]
 :TempDirectory:                              {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/TempDirectory.java[TempDirectory]
 :TestInfoParameterResolver:                  {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/TestInfoParameterResolver.java[TestInfoParameterResolver]
 :TestReporterParameterResolver:              {current-branch}/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/extension/TestReporterParameterResolver.java[TestReporterParameterResolver]

documentation/src/docs/asciidoc/release-notes/index.adoc

@@ -18,6 +18,8 @@ include::{includedir}/link-attributes.adoc[]
 
 include::{basedir}/release-notes-5.10.0-M1.adoc[]
 
+include::{basedir}/release-notes-5.9.3.adoc[]
+
 include::{basedir}/release-notes-5.9.2.adoc[]
 
 include::{basedir}/release-notes-5.9.1.adoc[]

documentation/src/docs/asciidoc/release-notes/release-notes-5.10.0-M1.adoc

@@ -24,6 +24,10 @@ repository on GitHub.
 
 ==== New Features and Improvements
 
+* Promote various "experimental" APIs that have matured to "stable" including
+  `ModuleSelector`, `EngineDiscoveryListener`, `EngineDiscoveryRequestResolver`,
+  `LauncherSession`, `LauncherSessionListener`, parallel execution support classes,
+  `@Suite` and related annotations, and others.
 * All utility methods in `ReflectionSupport` that return a `List` now have counterparts
   which return a `Stream`.
 * For consistency with JUnit Jupiter lifecycle callbacks, listener method pairs for
@@ -38,6 +42,7 @@ repository on GitHub.
   methods of the former. Please refer to the
   <<../user-guide/index.adoc#launcher-api-launcher-interceptors-custom, User Guide>> for
   details.
+* Support for limiting the `max-pool-size-factor` for parallel execution via a configuration parameter.
 * The new `testfeed` details mode for `ConsoleLauncher` prints test execution events as
   they occur in a concise format.
 * New `--exclude-methodname` and `--include-methodname` options added to the
@@ -51,34 +56,36 @@ will exclude all methods called `methodName` under package `org.example`.
 
 ==== Bug Fixes
 
-* Parameter types for _local_ `@MethodSource` factory method names are now validated. For
-  example, `@MethodSource("myFactory(example.NonexistentType)")` will now result in an
-  exception stating that `example.NonexistentType` cannot be resolved to a valid type.
-* The syntax for parameter types in _local_ `@MethodSource` factory method names now
-  supports canonical array names -- for example, you may now specify `int[]` as in
-  `@MethodSource("myFactory(int[])"` instead of the _binary_ name `[I` as in
-  `@MethodSource("myFactory([I)"` (which was already supported) and
-  `@MethodSource("myFactory(java.lang.String[])` instead of
-  `@MethodSource("myFactory([Ljava.lang.String;)`.
-* Exceptions thrown for undeletable files when cleaning up a temporary directory created
-  via `@TempDir` now include the root cause.
+* ❓
 
 ==== Deprecations and Breaking Changes
 
-* ❓
+* The `dynamic` parallel execution strategy now allows the thread pool to be saturated by
+default.
 
 ==== New Features and Improvements
 
+* Promote various "experimental" APIs that have matured to "stable" including
+  `MethodOrderer`, `ClassOrderer`, `InvocationInterceptor`,
+  `LifecycleMethodExecutionExceptionHandler`, `@TempDir`, parallel execution annotations,
+  and others.
+* `@RepeatedTest` can now be configured with a failure threshold which signifies the
+  number of failures after which remaining repetitions will be automatically skipped. See
+  the <<../user-guide/index.adoc#writing-tests-repeated-tests, User Guide>> for details.
 * New `ArgumentsAccessor.getInvocationIndex()` method that supplies the index of a
   `@ParameterizedTest` invocation.
-* `DisplayNameGenerator` methods are now allowed to return `null`, in order to signal to
-  fall back to the default display name generator.
 * `@EmptySource` now supports additional types, including `Collection` and `Map` subtypes
   with a public no-arg constructor.
+* `DisplayNameGenerator` methods are now allowed to return `null`, in order to signal to
+  fall back to the default display name generator.
 * New `AnnotationBasedArgumentsProvider` convenience base class which implements both
   `ArgumentsProvider` and `AnnotationConsumer`.
 * New `AnnotationBasedArgumentConverter` convenience base class which implements both
   `ArgumentConverter` and `AnnotationConsumer`.
+* New `junit.jupiter.execution.parallel.config.dynamic.max-pool-size-factor` configuration
+  parameter to set the maximum pool size factor.
+* New `junit.jupiter.execution.parallel.config.dynamic.saturate` configuration
+  parameter to disable pool saturation.
 
 
 [[release-notes-5.10.0-M1-junit-vintage]]

documentation/src/docs/asciidoc/release-notes/release-notes-5.9.3.adoc

@@ -0,0 +1,77 @@
+[[release-notes-5.9.3]]
+== 5.9.3
+
+*Date of Release:* ❓
+
+*Scope:* ❓
+
+For a complete list of all _closed_ issues and pull requests for this release, consult the
+link:{junit5-repo}+/milestone/67?closed=1+[5.9.3] milestone page in the
+JUnit repository on GitHub.
+
+
+[[release-notes-5.9.3-junit-platform]]
+=== JUnit Platform
+
+==== Bug Fixes
+
+* ❓
+
+==== Deprecations and Breaking Changes
+
+* ❓
+
+==== New Features and Improvements
+
+* ❓
+
+
+[[release-notes-5.9.3-junit-jupiter]]
+=== JUnit Jupiter
+
+==== Bug Fixes
+
+* Parameter types for _local_ `@MethodSource` factory method names are now validated. For
+  example, `@MethodSource("myFactory(example.NonexistentType)")` will now result in an
+  exception stating that `example.NonexistentType` cannot be resolved to a valid type.
+* The syntax for parameter types in _local_ `@MethodSource` factory method names now
+  supports canonical array names -- for example, you may now specify `int[]` as in
+  `@MethodSource("myFactory(int[])"` instead of the _binary_ name `[I` as in
+  `@MethodSource("myFactory([I)"` (which was already supported) and
+  `@MethodSource("myFactory(java.lang.String[])` instead of
+  `@MethodSource("myFactory([Ljava.lang.String;)`.
+* The search algorithm used to find `@MethodSource` factory methods now applies consistent
+  semantics for _local_ qualified method names and fully-qualified method names for
+  overloaded factory methods.
+* Exceptions thrown for files that cannot be deleted when cleaning up a temporary
+  directory created via `@TempDir` now include the root cause.
+* Lifecycle methods are allowed to be declared as `private` again for backwards
+  compatibility; however, using `private` visibility for lifecycle methods is strongly
+  discouraged and will be disallowed in a future release.
+
+==== Deprecations and Breaking Changes
+
+* ❓
+
+==== New Features and Improvements
+
+* The search algorithm used to find `@MethodSource` factory methods now falls back to
+  lenient search semantics when a factory method cannot be found by qualified name
+  (without a parameter list) and also provides better diagnostics when a unique factory
+  method cannot be found.
+
+
+[[release-notes-5.9.3-junit-vintage]]
+=== JUnit Vintage
+
+==== Bug Fixes
+
+* ❓
+
+==== Deprecations and Breaking Changes
+
+* ❓
+
+==== New Features and Improvements
+
+* ❓

documentation/src/docs/asciidoc/user-guide/writing-tests.adoc

@@ -862,13 +862,13 @@ following demonstrates how to have `TestInfo` injected into a test constructor,
 include::{testDir}/example/TestInfoDemo.java[tags=user_guide]
 ----
 
-* `{RepetitionInfoParameterResolver}`: if a method parameter in a `@RepeatedTest`,
-  `@BeforeEach`, or `@AfterEach` method is of type `{RepetitionInfo}`, the
-  `RepetitionInfoParameterResolver` will supply an instance of `RepetitionInfo`.
-  `RepetitionInfo` can then be used to retrieve information about the current repetition
-  and the total number of repetitions for the corresponding `@RepeatedTest`. Note,
-  however, that `RepetitionInfoParameterResolver` is not registered outside the context
-  of a `@RepeatedTest`. See <<writing-tests-repeated-tests-examples>>.
+* `{RepetitionExtension}`: if a method parameter in a `@RepeatedTest`, `@BeforeEach`, or
+  `@AfterEach` method is of type `{RepetitionInfo}`, the `RepetitionExtension` will supply
+  an instance of `RepetitionInfo`. `RepetitionInfo` can then be used to retrieve
+  information about the current repetition, the total number of repetitions, the number of
+  repetitions that have failed, and the failure threshold for the corresponding
+  `@RepeatedTest`. Note, however, that `RepetitionExtension` is not registered outside the
+  context of a `@RepeatedTest`. See <<writing-tests-repeated-tests-examples>>.
 
 * `{TestReporterParameterResolver}`: if a constructor or method parameter is of type
   `{TestReporter}`, the `TestReporterParameterResolver` will supply an instance of
@@ -1020,10 +1020,32 @@ void repeatedTest() {
 }
 ----
 
-In addition to specifying the number of repetitions, a custom display name can be
-configured for each repetition via the `name` attribute of the `@RepeatedTest`
-annotation. Furthermore, the display name can be a pattern composed of a combination of
-static text and dynamic placeholders. The following placeholders are currently supported.
+Since JUnit Jupiter 5.10, `@RepeatedTest` can be configured with a failure threshold which
+signifies the number of failures after which remaining repetitions will be automatically
+skipped. Set the `failureThreshold` attribute to a positive number less than the total
+number of repetitions in order to skip the invocations of remaining repetitions after the
+specified number of failures has been encountered.
+
+For example, if you are using `@RepeatedTest` to repeatedly invoke a test that you suspect
+to be _flaky_, a single failure is sufficient to demonstrate that the test is flaky, and
+there is no need to invoke the remaining repetitions. To support that specific use case,
+set `failureThreshold = 1`. You can alternatively set the threshold to a number greater
+than 1 depending on your use case.
+
+By default, the `failureThreshold` attribute is set to `Integer.MAX_VALUE`, signaling that
+no failure threshold will be applied, which effectively means that the specified number of
+repetitions will be invoked regardless of whether any repetitions fail.
+
+WARNING: If the repetitions of a `@RepeatedTest` method are executed in parallel, no
+guarantees can be made regarding the failure threshold. It is therefore recommended that a
+`@RepeatedTest` method be annotated with `@Execution(SAME_THREAD)` when parallel execution
+is configured. See <<writing-tests-parallel-execution>> for further details.
+
+In addition to specifying the number of repetitions and failure threshold, a custom
+display name can be configured for each repetition via the `name` attribute of the
+`@RepeatedTest` annotation. Furthermore, the display name can be a pattern composed of a
+combination of static text and dynamic placeholders. The following placeholders are
+currently supported.
 
 - `{displayName}`: display name of the `@RepeatedTest` method
 - `{currentRepetition}`: the current repetition count
@@ -1039,21 +1061,26 @@ latter is equal to `"{displayName} :: repetition {currentRepetition} of
 {totalRepetitions}"` which results in display names for individual repetitions like
 `repeatedTest() :: repetition 1 of 10`, `repeatedTest() :: repetition 2 of 10`, etc.
 
-In order to retrieve information about the current repetition and the total number of
-repetitions programmatically, a developer can choose to have an instance of
-`RepetitionInfo` injected into a `@RepeatedTest`, `@BeforeEach`, or `@AfterEach` method.
+In order to retrieve information about the current repetition, the total number of
+repetitions, the number of repetitions that have failed, and the failure threshold, a
+developer can choose to have an instance of `{RepetitionInfo}` injected into a
+`@RepeatedTest`, `@BeforeEach`, or `@AfterEach` method.
 
 [[writing-tests-repeated-tests-examples]]
 ==== Repeated Test Examples
 
 The `RepeatedTestsDemo` class at the end of this section demonstrates several examples of
 repeated tests.
 
-The `repeatedTest()` method is identical to example from the previous section; whereas,
+The `repeatedTest()` method is identical to the example from the previous section; whereas,
 `repeatedTestWithRepetitionInfo()` demonstrates how to have an instance of
 `RepetitionInfo` injected into a test to access the total number of repetitions for the
 current repeated test.
 
+`repeatedTestWithFailureThreshold()` demonstrates how to set a failure threshold and
+simulates an unexpected failure for every second repetition. The resulting behavior can be
+viewed in the `ConsoleLauncher` output at the end of this section.
+
 The next two methods demonstrate how to include a custom `@DisplayName` for the
 `@RepeatedTest` method in the display name of each repetition. `customDisplayName()`
 combines a custom display name with a custom pattern and then uses `TestInfo` to verify
@@ -1089,6 +1116,10 @@ INFO: About to execute repetition 2 of 5 for repeatedTestWithRepetitionInfo
 INFO: About to execute repetition 3 of 5 for repeatedTestWithRepetitionInfo
 INFO: About to execute repetition 4 of 5 for repeatedTestWithRepetitionInfo
 INFO: About to execute repetition 5 of 5 for repeatedTestWithRepetitionInfo
+INFO: About to execute repetition 1 of 8 for repeatedTestWithFailureThreshold
+INFO: About to execute repetition 2 of 8 for repeatedTestWithFailureThreshold
+INFO: About to execute repetition 3 of 8 for repeatedTestWithFailureThreshold
+INFO: About to execute repetition 4 of 8 for repeatedTestWithFailureThreshold
 INFO: About to execute repetition 1 of 1 for customDisplayName
 INFO: About to execute repetition 1 of 1 for customDisplayNameWithLongPattern
 INFO: About to execute repetition 1 of 5 for repeatedTestInGerman
@@ -1125,6 +1156,15 @@ When using the `ConsoleLauncher` with the unicode theme enabled, execution of
 │  │  ├─ repetition 3 of 5 ✔
 │  │  ├─ repetition 4 of 5 ✔
 │  │  └─ repetition 5 of 5 ✔
+│  ├─ repeatedTestWithFailureThreshold(RepetitionInfo) ✔
+│  │  ├─ repetition 1 of 8 ✔
+│  │  ├─ repetition 2 of 8 ✘ Boom!
+│  │  ├─ repetition 3 of 8 ✔
+│  │  ├─ repetition 4 of 8 ✘ Boom!
+│  │  ├─ repetition 5 of 8 ↷ Failure threshold [2] exceeded
+│  │  ├─ repetition 6 of 8 ↷ Failure threshold [2] exceeded
+│  │  ├─ repetition 7 of 8 ↷ Failure threshold [2] exceeded
+│  │  └─ repetition 8 of 8 ↷ Failure threshold [2] exceeded
 │  ├─ Repeat! ✔
 │  │  └─ Repeat! 1/1 ✔
 │  ├─ Details... ✔
@@ -2355,6 +2395,8 @@ configuration parameter to one of the following options.
   Computes the desired parallelism based on the number of available processors/cores
   multiplied by the `junit.jupiter.execution.parallel.config.dynamic.factor`
   configuration parameter (defaults to `1`).
+  The optional `junit.jupiter.execution.parallel.config.dynamic.max-pool-size-factor`
+  configuration parameter can be used to limit the maximum number of threads.
 
 `fixed`::
   Uses the mandatory `junit.jupiter.execution.parallel.config.fixed.parallelism`
@@ -2378,8 +2420,8 @@ of the synchronization mechanisms described in the next section, the `ForkJoinPo
 is used behind the scenes may spawn additional threads to ensure execution continues with
 sufficient parallelism.
 If you require such guarantees, with Java 9+, it is possible to limit the maximum number
-of concurrent threads by controlling the maximum pool size of the `fixed` and `custom`
-strategies.
+of concurrent threads by controlling the maximum pool size of the `dynamic`, `fixed` and
+`custom` strategies.
 
 [[writing-tests-parallel-execution-config-properties]]
 ===== Relevant properties
@@ -2426,6 +2468,22 @@ The following table lists relevant properties for configuring parallel execution
 | a positive decimal number
 | ```1.0```
 
+| ```junit.jupiter.execution.parallel.config.dynamic.max-pool-size-factor```
+| Factor to be multiplied by the number of available processors/cores and the value of
+  `junit.jupiter.execution.parallel.config.dynamic.factor` to determine the desired
+  parallelism for the ```dynamic``` configuration strategy
+| a positive decimal number, must be greater than or equal to `1.0`
+| 256 + the value of `junit.jupiter.execution.parallel.config.dynamic.factor` multiplied
+  by the number of available processors/cores
+
+| ```junit.jupiter.execution.parallel.config.dynamic.saturate```
+| Disable saturation of the underlying fork-join pool for the ```dynamic``` configuration
+strategy
+|
+* `true`
+* `false`
+| ```true```
+
 | ```junit.jupiter.execution.parallel.config.fixed.parallelism```
 | Desired parallelism for the ```fixed``` configuration strategy
 | a positive integer
@@ -2434,7 +2492,7 @@ The following table lists relevant properties for configuring parallel execution
 | ```junit.jupiter.execution.parallel.config.fixed.max-pool-size```
 | Desired maximum pool size of the underlying fork-join pool for the ```fixed```
   configuration strategy
-| a positive integer, must greater than or equal to `junit.jupiter.execution.parallel.config.fixed.parallelism`
+| a positive integer, must be greater than or equal to `junit.jupiter.execution.parallel.config.fixed.parallelism`
 | 256 + the value of `junit.jupiter.execution.parallel.config.fixed.parallelism`
 
 | ```junit.jupiter.execution.parallel.config.fixed.saturate```

documentation/src/test/java/example/RepeatedTestsDemo.java

@@ -12,6 +12,7 @@
 
 // tag::user_guide[]
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 import java.util.logging.Logger;
 
@@ -21,6 +22,10 @@
 import org.junit.jupiter.api.RepetitionInfo;
 import org.junit.jupiter.api.TestInfo;
 
+// end::user_guide[]
+// Use fully-qualified names to avoid having them show up in the imports.
+@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD)
+// tag::user_guide[]
 class RepeatedTestsDemo {
 
 	private Logger logger = // ...
@@ -47,6 +52,18 @@ void repeatedTestWithRepetitionInfo(RepetitionInfo repetitionInfo) {
 		assertEquals(5, repetitionInfo.getTotalRepetitions());
 	}
 
+	// end::user_guide[]
+	// Use fully-qualified name to avoid having it show up in the imports.
+	@org.junit.jupiter.api.Disabled("intentional failures would break the build")
+	// tag::user_guide[]
+	@RepeatedTest(value = 8, failureThreshold = 2)
+	void repeatedTestWithFailureThreshold(RepetitionInfo repetitionInfo) {
+		// Simulate unexpected failure every second repetition
+		if (repetitionInfo.getCurrentRepetition() % 2 == 0) {
+			fail("Boom!");
+		}
+	}
+
 	@RepeatedTest(value = 1, name = "{displayName} {currentRepetition}/{totalRepetitions}")
 	@DisplayName("Repeat!")
 	void customDisplayName(TestInfo testInfo) {

gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts

@@ -82,4 +82,7 @@ dependencies {
 	testRuntimeOnly(project(":junit-platform-reporting"))
 
 	testRuntimeOnly(bundleFromLibs("log4j"))
+	testRuntimeOnly(dependencyFromLibs("openTestReporting-events")) {
+		because("it's required to run tests via IntelliJ which does not consumed the shadowed jar of junit-platform-reporting")
+	}
 }