Skip to content

Commit

Permalink
[SUREFIRE-1345] - Support flakyFailure and flakyError in TestSuiteXml…
Browse files Browse the repository at this point in the history 
…Parser
  • Loading branch information
@gsmet
gsmet committed Dec 12, 2023
1 parent 484ef56 commit 970376d
Show file tree
Hide file tree
Showing 4 changed files with 652 additions and 0 deletions.
127 changes: 127 additions & 0 deletions ...-report-parser/src/main/java/org/apache/maven/plugins/surefire/report/ReportTestCase.java
View file
Expand Up @@ -18,6 +18,9 @@
*/
package org.apache.maven.plugins.surefire.report;

import java.util.ArrayList;
import java.util.List;

import static org.apache.maven.shared.utils.StringUtils.isNotBlank;

/**
Expand Down Expand Up @@ -48,6 +51,10 @@ public final class ReportTestCase {

private boolean hasSkipped;

private List<FlakyError> flakyErrors = new ArrayList<>();

private List<FlakyFailure> flakyFailures = new ArrayList<>();

public String getName() {
return name;
}
Expand Down Expand Up @@ -161,6 +168,26 @@ public boolean hasSkipped() {
return hasSkipped;
}

public boolean hasFlake() {
return !flakyErrors.isEmpty() || !flakyFailures.isEmpty();
}

public void addFlakyError(FlakyError flakyError) {
flakyErrors.add(flakyError);
}

public List<FlakyError> getFlakyErrors() {
return flakyErrors;
}

public void addFlakyFailure(FlakyFailure flakyFailure) {
flakyFailures.add(flakyFailure);
}

public List<FlakyFailure> getFlakyFailures() {
return flakyFailures;
}

/**
* {@inheritDoc}
*/
Expand All @@ -173,4 +200,104 @@ private ReportTestCase setFailureType(String failureType) {
this.failureType = failureType;
return this;
}

public static class FlakyFailure {

private String message;

private String type;

private String systemOut;

private String systemErr;

private String stackTrace;

FlakyFailure(String message, String type) {
this.message = message;
this.type = type;
}

public String getMessage() {
return message;
}

public String getType() {
return type;
}

public void setSystemOut(String systemOut) {
this.systemOut = systemOut;
}

public String getSystemOut() {
return systemOut;
}

public void setSystemErr(String systemErr) {
this.systemErr = systemErr;
}

public String getSystemErr() {
return systemErr;
}

public void setStackTrace(String stackTrace) {
this.stackTrace = stackTrace;
}

public String getStackTrace() {
return stackTrace;
}
}

public static class FlakyError {

private String message;

private String type;

private String systemOut;

private String systemErr;

private String stackTrace;

FlakyError(String message, String type) {
this.message = message;
this.type = type;
}

public String getMessage() {
return message;
}

public String getType() {
return type;
}

public void setSystemOut(String systemOut) {
this.systemOut = systemOut;
}

public String getSystemOut() {
return systemOut;
}

public void setSystemErr(String systemErr) {
this.systemErr = systemErr;
}

public String getSystemErr() {
return systemErr;
}

public void setStackTrace(String stackTrace) {
this.stackTrace = stackTrace;
}

public String getStackTrace() {
return stackTrace;
}
}
}
56 changes: 56 additions & 0 deletions ...ort-parser/src/main/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParser.java
View file
Expand Up @@ -58,6 +58,10 @@ public final class TestSuiteXmlParser extends DefaultHandler {

private ReportTestCase testCase;

private ReportTestCase.FlakyFailure testCaseFlakyFailure;

private ReportTestCase.FlakyError testCaseFlakyError;

private boolean valid;

private boolean parseContent;
Expand Down Expand Up @@ -131,6 +135,8 @@ public void startElement(String uri, String localName, String qName, Attributes
currentElement = new StringBuilder();

testCase = new ReportTestCase().setName(attributes.getValue("name"));
testCaseFlakyFailure = null;
testCaseFlakyError = null;

String fullClassName = attributes.getValue("classname");

Expand Down Expand Up @@ -177,9 +183,27 @@ public void startElement(String uri, String localName, String qName, Attributes
currentSuite.incrementNumberOfSkipped();
break;
case "flakyFailure":
testCaseFlakyFailure = new ReportTestCase.FlakyFailure(
attributes.getValue("message"), attributes.getValue("type"));
currentSuite.incrementNumberOfFlakes();
break;
case "flakyError":
testCaseFlakyError = new ReportTestCase.FlakyError(
attributes.getValue("message"), attributes.getValue("type"));
currentSuite.incrementNumberOfFlakes();
break;
case "stackTrace":
currentElement = new StringBuilder();
parseContent = true;
break;
case "system-out":
currentElement = new StringBuilder();
parseContent = true;
break;
case "system-err":
currentElement = new StringBuilder();
parseContent = true;
break;
case "failsafe-summary":
valid = false;
break;
Expand Down Expand Up @@ -217,6 +241,38 @@ public void endElement(String uri, String localName, String qName) throws SAXExc
throw new SAXException("Failed to parse time value", e);
}
break;
case "flakyFailure":
testCase.addFlakyFailure(testCaseFlakyFailure);
testCaseFlakyFailure = null;
break;
case "flakyError":
testCase.addFlakyError(testCaseFlakyError);
testCaseFlakyError = null;
break;
case "stackTrace":
if (testCaseFlakyFailure != null) {
testCaseFlakyFailure.setStackTrace(currentElement.toString());
}
if (testCaseFlakyError != null) {
testCaseFlakyError.setStackTrace(currentElement.toString());
}
break;
case "system-out":
if (testCaseFlakyFailure != null) {
testCaseFlakyFailure.setSystemOut(currentElement.toString());
}
if (testCaseFlakyError != null) {
testCaseFlakyError.setSystemOut(currentElement.toString());
}
break;
case "system-err":
if (testCaseFlakyFailure != null) {
testCaseFlakyFailure.setSystemErr(currentElement.toString());
}
if (testCaseFlakyError != null) {
testCaseFlakyError.setSystemErr(currentElement.toString());
}
break;
default:
break;
}
Expand Down
106 changes: 106 additions & 0 deletions ...parser/src/test/java/org/apache/maven/plugins/surefire/report/TestSuiteXmlParserTest.java
View file
Expand Up @@ -26,14 +26,18 @@
import java.util.List;

import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
import org.apache.maven.plugins.surefire.report.ReportTestCase.FlakyError;
import org.apache.maven.plugins.surefire.report.ReportTestCase.FlakyFailure;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
Expand Down Expand Up @@ -708,4 +712,106 @@ public void shouldParseLargeFile() throws Exception {
// Delete test file
Files.delete(tempFile);
}

@Test
public void shouldParseFlakes() throws Exception {
// Parse test file
TestSuiteXmlParser parser = new TestSuiteXmlParser(consoleLogger);

List<ReportTestSuite> testSuites =
parser.parse("src/test/resources/fixture/testsuitexmlparser/TEST-org.acme.FlakyTest.xml");
assertTrue(parser.isValid());
assertThat(testSuites.size(), is(1));

assertThat(testSuites.size(), is(1));
ReportTestSuite report = testSuites.get(0);
assertThat(report.getFullClassName(), is("org.acme.FlakyTest"));
assertThat(report.getName(), is("FlakyTest"));
assertThat(report.getPackageName(), is("org.acme"));
assertThat(report.getNumberOfTests(), is(2));
assertThat(report.getNumberOfSkipped(), is(0));
assertThat(report.getNumberOfErrors(), is(0));
assertThat(report.getNumberOfFailures(), is(0));
assertThat(report.getNumberOfFlakes(), is(4));
assertThat(report.getTimeElapsed(), is(1.324f));
assertThat(report.getTestCases().size(), is(2));

List<ReportTestCase> tests = report.getTestCases();
assertThat(tests.get(0).getFullClassName(), is("org.acme.FlakyTest"));
assertThat(tests.get(0).getName(), is("testFlaky"));
assertNull(tests.get(0).getFailureDetail());
assertThat(tests.get(0).getClassName(), is("FlakyTest"));
assertThat(tests.get(0).getTime(), is(0.034f));
assertThat(tests.get(0).getFullName(), is("org.acme.FlakyTest.testFlaky"));
assertThat(tests.get(0).hasError(), is(false));
assertThat(tests.get(0).hasFlake(), is(true));

List<FlakyFailure> flakyFailures = tests.get(0).getFlakyFailures();
assertThat(flakyFailures.size(), is(3));

assertThat(flakyFailures.get(0).getMessage(), startsWith("expected: <1> but was: <0>"));
assertThat(flakyFailures.get(0).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyFailures.get(0).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));
assertThat(
flakyFailures.get(0).getSystemOut(),
is(
"2023-06-23 17:27:31,351 INFO [io.quarkus] (main) code-with-quarkus 1.0.0-SNAPSHOT on JVM (powered by Quarkus 3.1.2.Final) started in 1.493s. Listening on: http://localhost:8081\n"
+ "2023-06-23 17:27:31,353 INFO [io.quarkus] (main) Profile test activated. \n"
+ "2023-06-23 17:27:31,353 INFO [io.quarkus] (main) Installed features: [cdi, resteasy-reactive, smallrye-context-propagation, vertx]\n"));
assertNull(flakyFailures.get(0).getSystemErr());

assertThat(flakyFailures.get(1).getMessage(), startsWith("expected: <1> but was: <3>"));
assertThat(flakyFailures.get(1).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyFailures.get(1).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));
assertThat(
flakyFailures.get(1).getSystemOut(),
is(
"2023-06-23 17:27:31,695 INFO [io.quarkus] (main) code-with-quarkus stopped in 0.022s\n"
+ "2023-06-23 17:27:33,059 INFO [io.quarkus] (main) code-with-quarkus 1.0.0-SNAPSHOT on JVM (powered by Quarkus 3.1.2.Final) started in 1.200s. Listening on: http://localhost:8081\n"
+ "2023-06-23 17:27:33,059 INFO [io.quarkus] (main) Profile test activated. \n"
+ "2023-06-23 17:27:33,059 INFO [io.quarkus] (main) Installed features: [cdi, resteasy-reactive, smallrye-context-propagation, vertx]\n"));
assertNull(flakyFailures.get(1).getSystemErr());

assertThat(flakyFailures.get(2).getMessage(), startsWith("expected: <1> but was: <4>"));
assertThat(flakyFailures.get(2).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyFailures.get(2).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));
assertThat(
flakyFailures.get(2).getSystemOut(),
is(
"2023-06-23 17:27:33,117 INFO [io.quarkus] (main) code-with-quarkus stopped in 0.016s\n"
+ "2023-06-23 17:27:34,404 INFO [io.quarkus] (main) code-with-quarkus 1.0.0-SNAPSHOT on JVM (powered by Quarkus 3.1.2.Final) started in 1.180s. Listening on: http://localhost:8081\n"
+ "2023-06-23 17:27:34,404 INFO [io.quarkus] (main) Profile test activated. \n"
+ "2023-06-23 17:27:34,404 INFO [io.quarkus] (main) Installed features: [cdi, resteasy-reactive, smallrye-context-propagation, vertx]\n"));
assertNull(flakyFailures.get(2).getSystemErr());

List<FlakyError> flakyErrors = tests.get(0).getFlakyErrors();
assertThat(flakyErrors.size(), is(1));
assertThat(flakyErrors.get(0).getMessage(), startsWith("expected: <1> but was: <0>"));
assertThat(flakyErrors.get(0).getType(), is("org.opentest4j.AssertionFailedError"));
assertThat(
flakyErrors.get(0).getStackTrace(),
containsString("at org.acme.FlakyTest.testFlaky(FlakyTest.java:18)"));
assertThat(
flakyErrors.get(0).getSystemOut(),
is(
"2023-06-23 17:27:34,461 INFO [io.quarkus] (main) code-with-quarkus stopped in 0.014s\n"
+ "2023-06-23 17:27:35,705 INFO [io.quarkus] (main) code-with-quarkus 1.0.0-SNAPSHOT on JVM (powered by Quarkus 3.1.2.Final) started in 1.159s. Listening on: http://localhost:8081\n"
+ "2023-06-23 17:27:35,705 INFO [io.quarkus] (main) Profile test activated. \n"
+ "2023-06-23 17:27:35,705 INFO [io.quarkus] (main) Installed features: [cdi, resteasy-reactive, smallrye-context-propagation, vertx]\n"));
assertThat(flakyErrors.get(0).getSystemErr(), is("Test system.err\n"));

assertThat(tests.get(1).getFullClassName(), is("org.acme.FlakyTest"));
assertThat(tests.get(1).getName(), is("testStable"));
assertThat(tests.get(1).getClassName(), is("FlakyTest"));
assertThat(tests.get(1).getTime(), is(0.001f));
assertThat(tests.get(1).getFullName(), is("org.acme.FlakyTest.testStable"));
assertThat(tests.get(1).hasError(), is(false));
assertThat(tests.get(1).hasFlake(), is(false));
}
}

0 comments on commit 970376d

Please sign in to comment.