Skip to content

Commit

Permalink
Merge pull request #23959 from stuartwdouglas/23937
Browse files Browse the repository at this point in the history 
Re-run failed parameterized tests
  • Loading branch information
@Sanne
Sanne committed Feb 25, 2022
2 parents baa2410 + d0fb15b commit 77a14d4
Show file tree
Hide file tree
Showing 8 changed files with 59 additions and 47 deletions.
5 changes: 0 additions & 5 deletions ...src/main/java/io/quarkus/deployment/dev/testing/ContinuousTestingSharedStateListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,6 @@ public void runComplete(TestRunResults testRunResults) {
});
}

@Override
public void noTests(TestRunResults results) {
runComplete(results);
}

@Override
public void runAborted() {
ContinuousTestingSharedStateManager.setInProgress(false);
Expand Down
49 changes: 29 additions & 20 deletions core/deployment/src/main/java/io/quarkus/deployment/dev/testing/JunitTestRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,21 +178,6 @@ public FilterResult apply(TestDescriptor testDescriptor) {
LauncherDiscoveryRequest request = launchBuilder
.build();
TestPlan testPlan = launcher.discover(request);
if (!testPlan.containsTests()) {
testState.pruneDeletedTests(allDiscoveredIds, dynamicIds);
//nothing to see here
for (TestRunListener i : listeners) {
i.noTests(new TestRunResults(runId, classScanResult, classScanResult == null, start,
System.currentTimeMillis(), toResultsMap(testState.getCurrentResults())));
}
quarkusTestClasses.close();
return new Runnable() {
@Override
public void run() {

}
};
}
long toRun = testPlan.countTestIdentifiers(TestIdentifier::isTest);
for (TestRunListener listener : listeners) {
listener.runStarted(toRun);
Expand Down Expand Up @@ -226,13 +211,18 @@ public void quarkusStarting() {
});

Map<String, Map<UniqueId, TestResult>> resultsByClass = new HashMap<>();
AtomicReference<TestIdentifier> currentNonDynamicTest = new AtomicReference<>();
launcher.execute(testPlan, new TestExecutionListener() {

@Override
public void executionStarted(TestIdentifier testIdentifier) {
if (aborted) {
return;
}
boolean dynamic = dynamicIds.contains(UniqueId.parse(testIdentifier.getUniqueId()));
if (!dynamic) {
currentNonDynamicTest.set(testIdentifier);
}
startTimes.put(testIdentifier, System.currentTimeMillis());
String testClassName = "";
Class<?> testClass = getTestClassFromSource(testIdentifier.getSource());
Expand Down Expand Up @@ -260,7 +250,7 @@ public void executionSkipped(TestIdentifier testIdentifier, String reason) {
s -> new HashMap<>());
TestResult result = new TestResult(displayName, testClass.getName(), id,
TestExecutionResult.aborted(null),
logHandler.captureOutput(), testIdentifier.isTest(), runId, 0);
logHandler.captureOutput(), testIdentifier.isTest(), runId, 0, true);
results.put(id, result);
if (result.isTest()) {
for (TestRunListener listener : listeners) {
Expand All @@ -274,6 +264,9 @@ public void executionSkipped(TestIdentifier testIdentifier, String reason) {
@Override
public void dynamicTestRegistered(TestIdentifier testIdentifier) {
dynamicIds.add(UniqueId.parse(testIdentifier.getUniqueId()));
for (TestRunListener listener : listeners) {
listener.dynamicTestRegistered(testIdentifier);
}
}

@Override
Expand All @@ -282,6 +275,7 @@ public void executionFinished(TestIdentifier testIdentifier,
if (aborted) {
return;
}
boolean dynamic = dynamicIds.contains(UniqueId.parse(testIdentifier.getUniqueId()));
Set<String> touched = touchedClasses.pop();
Class<?> testClass = getTestClassFromSource(testIdentifier.getSource());
String displayName = getDisplayNameFromIdentifier(testIdentifier, testClass);
Expand All @@ -306,18 +300,33 @@ public void executionFinished(TestIdentifier testIdentifier,
testClassUsages.updateTestData(testClassName, id, touched);
}
}

Map<UniqueId, TestResult> results = resultsByClass.computeIfAbsent(testClassName,
s -> new HashMap<>());
TestResult result = new TestResult(displayName, testClassName, id,
testExecutionResult,
logHandler.captureOutput(), testIdentifier.isTest(), runId,
System.currentTimeMillis() - startTimes.get(testIdentifier));
results.put(id, result);
System.currentTimeMillis() - startTimes.get(testIdentifier), true);
if (!results.containsKey(id)) {
//if a child has failed we may have already marked the parent failed
results.put(id, result);
}
if (result.isTest()) {
for (TestRunListener listener : listeners) {
listener.testComplete(result);
}
if (dynamic && testExecutionResult.getStatus() == TestExecutionResult.Status.FAILED) {
//if it is dynamic we fail the parent as well for re-runs

RuntimeException failure = new RuntimeException("A child test failed");
failure.setStackTrace(new StackTraceElement[0]);
results.put(id,
new TestResult(currentNonDynamicTest.get().getDisplayName(),
result.getTestClass(),
currentNonDynamicTest.get().getUniqueIdObject(),
TestExecutionResult.failed(failure), List.of(), false, runId, 0,
false));
results.put(UniqueId.parse(currentNonDynamicTest.get().getUniqueId()), result);
}
} else if (testExecutionResult.getStatus() == TestExecutionResult.Status.FAILED) {
//if a parent fails we fail the children
Set<TestIdentifier> children = testPlan.getChildren(testIdentifier);
Expand All @@ -327,7 +336,7 @@ public void executionFinished(TestIdentifier testIdentifier,
childId,
testExecutionResult,
logHandler.captureOutput(), child.isTest(), runId,
System.currentTimeMillis() - startTimes.get(testIdentifier));
System.currentTimeMillis() - startTimes.get(testIdentifier), true);
results.put(childId, result);
if (child.isTest()) {
for (TestRunListener listener : listeners) {
Expand Down
18 changes: 10 additions & 8 deletions core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestConsoleHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,9 +220,11 @@ public void runComplete(TestRunResults results) {
log.error(statusHeader("TEST REPORT #" + results.getId()));
for (Map.Entry<String, TestClassResult> classEntry : results.getCurrentFailing().entrySet()) {
for (TestResult test : classEntry.getValue().getFailing()) {
log.error(
RED + "Test " + test.getDisplayName() + " failed \n" + RESET,
test.getTestExecutionResult().getThrowable().get());
if (test.isReportable()) {
log.error(
RED + "Test " + test.getDisplayName() + " failed \n" + RESET,
test.getTestExecutionResult().getThrowable().get());
}
}
}
log.error(
Expand All @@ -244,11 +246,6 @@ public void runComplete(TestRunResults results) {
}
}

@Override
public void noTests(TestRunResults results) {
runComplete(results);
}

@Override
public void runAborted() {
firstRun = false;
Expand All @@ -268,6 +265,11 @@ public void testStarted(TestIdentifier testIdentifier, String className) {
}
testsStatusOutput.setMessage(status);
}

@Override
public void dynamicTestRegistered(TestIdentifier testIdentifier) {
totalNoTests.incrementAndGet();
}
});

}
Expand Down
8 changes: 7 additions & 1 deletion core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ public class TestResult {
final long runId;
final long time;
final List<Throwable> problems;
final boolean reportable;

public TestResult(String displayName, String testClass, UniqueId uniqueId, TestExecutionResult testExecutionResult,
List<String> logOutput, boolean test, long runId, long time) {
List<String> logOutput, boolean test, long runId, long time, boolean reportable) {
this.displayName = displayName;
this.testClass = testClass;
this.uniqueId = uniqueId;
Expand All @@ -29,6 +30,7 @@ public TestResult(String displayName, String testClass, UniqueId uniqueId, TestE
this.test = test;
this.runId = runId;
this.time = time;
this.reportable = reportable;
List<Throwable> problems = new ArrayList<>();
if (testExecutionResult.getThrowable().isPresent()) {
Throwable t = testExecutionResult.getThrowable().get();
Expand Down Expand Up @@ -75,4 +77,8 @@ public long getTime() {
public List<Throwable> getProblems() {
return problems;
}

public boolean isReportable() {
return reportable;
}
}
2 changes: 1 addition & 1 deletion core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestRunListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ default void testStarted(TestIdentifier testIdentifier, String className) {

}

default void noTests(TestRunResults results) {
default void dynamicTestRegistered(TestIdentifier testIdentifier) {

}
}
12 changes: 0 additions & 12 deletions core/deployment/src/main/java/io/quarkus/deployment/dev/testing/TestSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,20 +374,8 @@ public void testStarted(TestIdentifier testIdentifier, String className) {
}
}

@Override
public void noTests(TestRunResults results) {
allResults.add(results);
runStarted(0);
}
}));
}
if (testCount.get() == 0) {
TestRunResults results = new TestRunResults(runId, classScanResult, classScanResult == null, start,
System.currentTimeMillis(), Collections.emptyMap());
for (var i : testRunListeners) {
i.noTests(results);
}
}
for (var i : testRunListeners) {
i.runStarted(testCount.get());
}
Expand Down
4 changes: 4 additions & 0 deletions ...ployment/src/main/java/io/quarkus/vertx/http/deployment/devmode/tests/TestsProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ DevConsoleRouteBuildItem runAllTests(LaunchModeBuildItem launchModeBuildItem) {
@Override
public void handle(RoutingContext event) {
ts.get().runAllTests();
event.response().setStatusCode(204).end();
}
});
}
Expand Down Expand Up @@ -178,6 +179,7 @@ DevConsoleRouteBuildItem runFailedTests(LaunchModeBuildItem launchModeBuildItem)
@Override
public void handle(RoutingContext event) {
ts.get().runFailedTests();
event.response().setStatusCode(204).end();
}
});
}
Expand All @@ -192,6 +194,7 @@ DevConsoleRouteBuildItem printfailures(LaunchModeBuildItem launchModeBuildItem)
@Override
public void handle(RoutingContext event) {
ts.get().printFullResults();
event.response().setStatusCode(204).end();
}
});
}
Expand Down Expand Up @@ -235,6 +238,7 @@ DevConsoleRouteBuildItem forceRestart(LaunchModeBuildItem launchModeBuildItem) {
@Override
public void handle(RoutingContext event) {
RuntimeUpdatesProcessor.INSTANCE.doScan(true, true);
event.response().setStatusCode(204).end();
}
});
}
Expand Down
8 changes: 8 additions & 0 deletions ...ment/src/test/java/io/quarkus/vertx/http/testrunner/params/TestParameterizedTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import io.quarkus.test.ContinuousTestingTestUtils;
import io.quarkus.test.ContinuousTestingTestUtils.TestStatus;
import io.quarkus.test.QuarkusDevModeTest;
import io.restassured.RestAssured;

public class TestParameterizedTestCase {

Expand Down Expand Up @@ -42,6 +43,13 @@ public void testParameterizedTests() throws InterruptedException {
Assertions.assertEquals(4L, ts.getTestsPassed());
Assertions.assertEquals(0L, ts.getTestsSkipped());

RestAssured.post("q/dev/io.quarkus.quarkus-vertx-http/tests/runfailed");

ts = utils.waitForNextCompletion();

Assertions.assertEquals(1L, ts.getTestsFailed());
Assertions.assertEquals(3L, ts.getTestsPassed()); //they are all re-run
Assertions.assertEquals(0L, ts.getTestsSkipped());
test.modifyTestSourceFile(ParamET.class, new Function<String, String>() {
@Override
public String apply(String s) {
Expand Down

0 comments on commit 77a14d4

Please sign in to comment.