From 0e5e7b3d459ef401af8a549e3ecfc123f87739ea Mon Sep 17 00:00:00 2001 From: Marc Philipp Date: Fri, 13 Dec 2024 18:09:09 +0100 Subject: [PATCH 1/3] Report captured output using OTR's new dedicated XML element --- documentation/documentation.gradle.kts | 2 ++ .../release-notes-5.12.0-M1.adoc | 1 + .../junit-platform-reporting.adoc | 3 ++ .../junitbuild.testing-conventions.gradle.kts | 2 ++ .../xml/OpenTestReportGeneratingListener.java | 27 +++++++++++++-- ...OpenTestReportGeneratingListenerTests.java | 33 +++++++++++++++++++ 6 files changed, 66 insertions(+), 2 deletions(-) diff --git a/documentation/documentation.gradle.kts b/documentation/documentation.gradle.kts index 761231335196..e5f5fb28fdb8 100644 --- a/documentation/documentation.gradle.kts +++ b/documentation/documentation.gradle.kts @@ -158,6 +158,8 @@ tasks { args.addAll("execute") args.addAll("--scan-classpath") args.addAll("--config=junit.platform.reporting.open.xml.enabled=true") + args.addAll("--config=junit.platform.output.capture.stdout=true") + args.addAll("--config=junit.platform.output.capture.stderr=true") outputs.dir(consoleLauncherTestReportsDir) argumentProviders.add(CommandLineArgumentProvider { listOf( diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc index 18904ee1545e..048ca30c3bee 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc @@ -47,6 +47,7 @@ JUnit repository on GitHub. - A section containing JUnit-specific metadata about each test/container to the HTML report is now written by open-test-reporting when added to the classpath/module path - Information about published files is now included as attachments. + - If output capturing is enabled, the captured output is now included in the XML report. * Introduced contracts for Kotlin-specific assertion methods. diff --git a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc index 310788bd1812..ef5725f90381 100644 --- a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc +++ b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc @@ -44,6 +44,9 @@ The listener is auto-registered and can be configured via the following If enabled, the listener creates an XML report file named `open-test-report.xml` in the configured <>. +If <> is enabled, it will be included in +the report as well. + TIP: The {OpenTestReportingCliTool} can be used to convert from the event-based format to the hierarchical format which is more human-readable. diff --git a/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts b/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts index f4834d0549aa..b9c14440fafe 100644 --- a/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts +++ b/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts @@ -126,6 +126,8 @@ tasks.withType().configureEach { jvmArgumentProviders += CommandLineArgumentProvider { listOf( "-Djunit.platform.reporting.open.xml.enabled=true", + "-Djunit.platform.output.capture.stdout=true", + "-Djunit.platform.output.capture.stderr=true", "-Djunit.platform.reporting.output.dir=${reports.junitXml.outputLocation.get().asFile.absolutePath}/junit-{uniqueNumber}", ) } diff --git a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListener.java b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListener.java index ba18a68c128b..82be764786fd 100644 --- a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListener.java +++ b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListener.java @@ -12,6 +12,8 @@ import static org.apiguardian.api.API.Status.EXPERIMENTAL; import static org.junit.platform.commons.util.StringUtils.isNotBlank; +import static org.junit.platform.launcher.LauncherConstants.STDERR_REPORT_ENTRY_KEY; +import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY; import static org.junit.platform.reporting.open.xml.JUnitFactory.legacyReportingName; import static org.junit.platform.reporting.open.xml.JUnitFactory.type; import static org.junit.platform.reporting.open.xml.JUnitFactory.uniqueId; @@ -25,6 +27,7 @@ import static org.opentest4j.reporting.events.core.CoreFactory.infrastructure; import static org.opentest4j.reporting.events.core.CoreFactory.metadata; import static org.opentest4j.reporting.events.core.CoreFactory.operatingSystem; +import static org.opentest4j.reporting.events.core.CoreFactory.output; import static org.opentest4j.reporting.events.core.CoreFactory.reason; import static org.opentest4j.reporting.events.core.CoreFactory.result; import static org.opentest4j.reporting.events.core.CoreFactory.sources; @@ -60,6 +63,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; +import java.time.LocalDateTime; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -90,6 +94,7 @@ import org.junit.platform.launcher.TestPlan; import org.opentest4j.reporting.events.api.DocumentWriter; import org.opentest4j.reporting.events.api.NamespaceRegistry; +import org.opentest4j.reporting.events.core.Attachments; import org.opentest4j.reporting.events.core.Infrastructure; import org.opentest4j.reporting.events.core.Result; import org.opentest4j.reporting.events.core.Sources; @@ -345,8 +350,26 @@ public void reportingEntryPublished(TestIdentifier testIdentifier, ReportEntry e String id = inProgressIds.get(testIdentifier.getUniqueIdObject()); eventsFileWriter.append(reported(id, Instant.now()), // reported -> reported.append(attachments(), // - attachments -> attachments.append(data(entry.getTimestamp()), // - data -> entry.getKeyValuePairs().forEach(data::addEntry)))); + attachments -> { + Map keyValuePairs = entry.getKeyValuePairs(); + if (keyValuePairs.containsKey(STDOUT_REPORT_ENTRY_KEY) + || keyValuePairs.containsKey(STDERR_REPORT_ENTRY_KEY)) { + attachOutput(attachments, entry.getTimestamp(), keyValuePairs.get(STDOUT_REPORT_ENTRY_KEY), + "stdout"); + attachOutput(attachments, entry.getTimestamp(), keyValuePairs.get(STDERR_REPORT_ENTRY_KEY), + "stderr"); + } + else { + attachments.append(data(entry.getTimestamp()), // + data -> keyValuePairs.forEach(data::addEntry)); + } + })); + } + + private static void attachOutput(Attachments attachments, LocalDateTime timestamp, String content, String source) { + if (content != null) { + attachments.append(output(timestamp), output -> output.withSource(source).withContent(content)); + } } @Override diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListenerTests.java b/platform-tests/src/test/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListenerTests.java index 4d1f9ebda5c5..b0efc6a5318e 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListenerTests.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/open/xml/OpenTestReportGeneratingListenerTests.java @@ -16,6 +16,8 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId; +import static org.junit.platform.launcher.LauncherConstants.CAPTURE_STDERR_PROPERTY_NAME; +import static org.junit.platform.launcher.LauncherConstants.CAPTURE_STDOUT_PROPERTY_NAME; import static org.junit.platform.launcher.LauncherConstants.OUTPUT_DIR_PROPERTY_NAME; import static org.junit.platform.launcher.LauncherConstants.OUTPUT_DIR_UNIQUE_NUMBER_PLACEHOLDER; import static org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder.request; @@ -23,11 +25,14 @@ import static org.junit.platform.reporting.open.xml.OpenTestReportGeneratingListener.ENABLED_PROPERTY_NAME; import static org.junit.platform.reporting.testutil.FileUtils.findPath; +import java.io.PrintStream; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -51,12 +56,32 @@ */ public class OpenTestReportGeneratingListenerTests { + private PrintStream originalOut; + private PrintStream originalErr; + + @BeforeEach + void wrapSystemPrintStreams() { + // Work around nesting check in org.junit.platform.launcher.core.StreamInterceptor + originalOut = System.out; + System.setOut(new PrintStream(System.out)); + originalErr = System.err; + System.setErr(new PrintStream(System.err)); + } + + @AfterEach + void restoreSystemPrintStreams() { + System.setOut(originalOut); + System.setErr(originalErr); + } + @Test void writesValidXmlReport(@TempDir Path tempDirectory) throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("failingTest", "display<-->Name 😎", (context, descriptor) -> { var listener = context.request.getEngineExecutionListener(); listener.reportingEntryPublished(descriptor, ReportEntry.from("key", "value")); + System.out.println("Hello, stdout!"); + System.err.println("Hello, stderr!"); fail("failure message"); }); @@ -105,6 +130,12 @@ void writesValidXmlReport(@TempDir Path tempDirectory) throws Exception { + + + + + + @@ -200,6 +231,8 @@ private void executeTests(Path tempDirectory, TestEngine engine, Path outputDir) var build = request() // .selectors(selectUniqueId(UniqueId.forEngine(engine.getId()))) // .configurationParameter(ENABLED_PROPERTY_NAME, String.valueOf(true)) // + .configurationParameter(CAPTURE_STDOUT_PROPERTY_NAME, String.valueOf(true)) // + .configurationParameter(CAPTURE_STDERR_PROPERTY_NAME, String.valueOf(true)) // .configurationParameter(OUTPUT_DIR_PROPERTY_NAME, outputDir.toString()) // .build(); createLauncher(engine).execute(build, new OpenTestReportGeneratingListener(tempDirectory)); From f7adb466498898b1d5332e96ecb4116f16cc10eb Mon Sep 17 00:00:00 2001 From: Marc Philipp Date: Mon, 16 Dec 2024 08:39:28 +0100 Subject: [PATCH 2/3] Improve docs --- .../asciidoc/release-notes/release-notes-5.12.0-M1.adoc | 8 ++++++-- .../advanced-topics/junit-platform-reporting.adoc | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc index 048ca30c3bee..8281d15a7315 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc @@ -41,13 +41,17 @@ JUnit repository on GitHub. allow test engines to publish/attach files to containers and tests by calling `EngineExecutionListener.fileEntryPublished(...)`. Registered `TestExecutionListeners` can then access these files by overriding the `fileEntryPublished(...)` method. -* The following improvements have been made to the open-test-reporting XML output: +* The following improvements have been made to the + <<../user-guide/index.adoc#junit-platform-reporting-open-test-reporting, Open Test Reporting>> + XML output: - Information about the Git repository, the current branch, the commit hash, and the current worktree status are now included in the XML report, if applicable. - A section containing JUnit-specific metadata about each test/container to the HTML report is now written by open-test-reporting when added to the classpath/module path - Information about published files is now included as attachments. - - If output capturing is enabled, the captured output is now included in the XML report. + - If <<../user-guide/index.adoc#running-tests-capturing-output, output capturing>> is + enabled, the captured output written to `System.out` and `System.err` is now included + in the XML report. * Introduced contracts for Kotlin-specific assertion methods. diff --git a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc index ef5725f90381..2e5b4d7369f1 100644 --- a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc +++ b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-reporting.adoc @@ -44,8 +44,8 @@ The listener is auto-registered and can be configured via the following If enabled, the listener creates an XML report file named `open-test-report.xml` in the configured <>. -If <> is enabled, it will be included in -the report as well. +If <> is enabled, the captured output +written to `System.out` and `System.err` will be included in the report as well. TIP: The {OpenTestReportingCliTool} can be used to convert from the event-based format to the hierarchical format which is more human-readable. From ed8bb494f0b764ef24d0262a8dab09e7a8c4203a Mon Sep 17 00:00:00 2001 From: Marc Philipp Date: Mon, 16 Dec 2024 08:42:30 +0100 Subject: [PATCH 3/3] Disable output capturing for integration tests --- .../main/kotlin/junitbuild.testing-conventions.gradle.kts | 4 ++-- .../platform-tooling-support-tests.gradle.kts | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts b/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts index b9c14440fafe..5ec321f5f5d4 100644 --- a/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts +++ b/gradle/plugins/common/src/main/kotlin/junitbuild.testing-conventions.gradle.kts @@ -126,11 +126,11 @@ tasks.withType().configureEach { jvmArgumentProviders += CommandLineArgumentProvider { listOf( "-Djunit.platform.reporting.open.xml.enabled=true", - "-Djunit.platform.output.capture.stdout=true", - "-Djunit.platform.output.capture.stderr=true", "-Djunit.platform.reporting.output.dir=${reports.junitXml.outputLocation.get().asFile.absolutePath}/junit-{uniqueNumber}", ) } + systemProperty("junit.platform.output.capture.stdout", "true") + systemProperty("junit.platform.output.capture.stderr", "true") jvmArgumentProviders += objects.newInstance(JavaAgentArgumentProvider::class).apply { classpath.from(javaAgentClasspath) diff --git a/platform-tooling-support-tests/platform-tooling-support-tests.gradle.kts b/platform-tooling-support-tests/platform-tooling-support-tests.gradle.kts index 88f991589508..9e5b70ba9ef2 100644 --- a/platform-tooling-support-tests/platform-tooling-support-tests.gradle.kts +++ b/platform-tooling-support-tests/platform-tooling-support-tests.gradle.kts @@ -165,6 +165,12 @@ tasks.test { dir("${rootDir}/documentation/src/test").withPathSensitivity(RELATIVE) } + // Disable capturing output since parallel execution is enabled and output of + // external processes happens on non-test threads which can't reliably be + // attributed to the test that started the process. + systemProperty("junit.platform.output.capture.stdout", "false") + systemProperty("junit.platform.output.capture.stderr", "false") + develocity { testDistribution { requirements.add("jdk=8")