Merge pull request #147 from veniosg/master

Fix double counting of fatal failures
shazam · Feb 19, 2019 · 30803a3 · 30803a3
2 parents 4068669 + 1767b43
commit 30803a3
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 18 deletions.
diff --git a/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java b/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java
@@ -81,9 +81,9 @@ public boolean run() {
                 reportMissingTests(aggregatedTestResult);
                 System.out.println("Scheduling their re-execution");
 
-                Collection<TestCaseEvent> testsToExecute =
+                Collection<TestCaseEvent> fatalCrashedTestCases =
                         findFatalCrashedTestCases(testCases, aggregatedTestResult.getFatalCrashedTests());
-                executeTests(poolExecutor, pools, testsToExecute);
+                executeTests(poolExecutor, pools, fatalCrashedTestCases);
 
                 aggregatedTestResult = aggregator.aggregateTestResults(pools, testCases);
 

diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java
@@ -72,14 +72,23 @@ public long millisSinceTestsStarted() {
     }
 
     @Override
-    public int getFailures() {
+    public int getTestFailures() {
         int sum = 0;
         for (PoolProgressTracker value : poolProgressTrackers.values()) {
             sum += value.getNumberOfFailedTests();
         }
         return sum;
     }
 
+    @Override
+    public int getTestRunFailures() {
+        int sum = 0;
+        for (PoolProgressTracker value : poolProgressTrackers.values()) {
+            sum += value.getNumberOfFailedTestRuns();
+        }
+        return sum;
+    }
+
     @Override
     public float getProgress() {
         float size = poolProgressTrackers.size();
@@ -137,8 +146,9 @@ private boolean totalAllowedRetryAvailable() {
         private void log(int testCaseFailures, boolean singleTestAllowed, boolean result) {
             logBuilder.setLength(0); //clean up.
             logBuilder.append("Retry requested ")
-                    .append(result ? " and allowed. " : " but not allowed. ")
-                    .append("Total retry left :").append(totalAllowedRetryLeft.get())
+                    .append(result ? "and allowed. " : "but not allowed. ")
+                    .append("Total retry left: ")
+                    .append(totalAllowedRetryLeft.get())
                     .append(" and Single Test case retry left: ")
                     .append(singleTestAllowed ? maxRetryPerTestCaseQuota - testCaseFailures : 0);
             logger.debug(logBuilder.toString());

diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTracker.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTracker.java
@@ -15,9 +15,13 @@ public interface PoolProgressTracker {
 
     void failedTest();
 
+    void failedTestRun();
+
     void trackTestEnqueuedAgain();
 
     float getProgress();
 
     int getNumberOfFailedTests();
+
+    int getNumberOfFailedTestRuns();
 }
diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTrackerImpl.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTrackerImpl.java
@@ -15,6 +15,7 @@ public class PoolProgressTrackerImpl implements PoolProgressTracker {
     private final int totalTests;
     private int failedTests;
     private int completedTests;
+    private int failedTestRuns;
 
     public PoolProgressTrackerImpl(int totalTests) {
         this.totalTests = totalTests;
@@ -30,6 +31,11 @@ public void failedTest() {
         failedTests++;
     }
 
+    @Override
+    public void failedTestRun() {
+        failedTestRuns++;
+    }
+
     @Override
     public void trackTestEnqueuedAgain() {
         completedTests--;
@@ -45,4 +51,9 @@ public float getProgress() {
     public int getNumberOfFailedTests() {
         return failedTests;
     }
+
+    @Override
+    public int getNumberOfFailedTestRuns() {
+        return failedTestRuns;
+    }
 }
diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java
@@ -30,7 +30,9 @@ public interface ProgressReporter {
      */
     long millisSinceTestsStarted();
 
-    int getFailures();
+    int getTestFailures();
+
+    int getTestRunFailures();
 
     float getProgress();
 

diff --git a/...-runner/src/main/java/com/shazam/fork/runner/listeners/ConsoleLoggingTestRunListener.java b/...-runner/src/main/java/com/shazam/fork/runner/listeners/ConsoleLoggingTestRunListener.java
@@ -45,14 +45,14 @@ class ConsoleLoggingTestRunListener extends NoOpITestRunListener {
 
     @Override
     public void testStarted(TestIdentifier test) {
-        System.out.println(format("%s %s %s %s [%s] %s", runningTime(), progress(), failures(), modelName,
-                serial, testCase(test)));
+        System.out.println(format("%s %s %s %s %s [%s] %s", runningTime(), progress(), failures(),
+                runFailures(), modelName, serial, testCase(test)));
     }
 
     @Override
     public void testFailed(TestIdentifier test, String trace) {
-        System.out.println(format("%s %s %s %s [%s] Failed %s\n %s", runningTime(), progress(), failures(), modelName,
-                serial, testCase(test), trace));
+        System.out.println(format("%s %s %s %s %s [%s] Failed %s\n %s", runningTime(), progress(),
+                failures(), runFailures(), modelName, serial, testCase(test), trace));
     }
 
     @Override
@@ -68,14 +68,14 @@ public void testIgnored(TestIdentifier test) {
 
     @Override
     public void testRunFailed(String errorMessage) {
-        System.out.println(format("%s %s %s %s [%s] Test run failed\n%s", runningTime(), progress(), failures(),
-                modelName, serial, errorMessage));
+        System.out.println(format("%s %s %s %s %s [%s] Test run failed\n%s", runningTime(),
+                progress(), failures(), runFailures(), modelName, serial, errorMessage));
     }
 
     @Override
     public void testRunStopped(long elapsedTime) {
-        System.out.println(format("%s %s %s %s [%s] Test run stopped after %s ms", runningTime(), progress(),
-                failures(), modelName, serial, elapsedTime));
+        System.out.println(format("%s %s %s %s %s [%s] Test run stopped after %s ms", runningTime(),
+                progress(), failures(), runFailures(), modelName, serial, elapsedTime));
     }
 
     private String runningTime() {
@@ -92,6 +92,10 @@ private String progress() {
     }
 
     private int failures() {
-        return progressReporter.getFailures();
+        return progressReporter.getTestFailures();
+    }
+
+    private int runFailures() {
+        return progressReporter.getTestRunFailures();
     }
 }
diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ProgressTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ProgressTestRunListener.java
@@ -19,7 +19,6 @@
 import java.util.Map;
 
 class ProgressTestRunListener implements ITestRunListener {
-
     private final PoolProgressTracker poolProgressTracker;
 
     ProgressTestRunListener(Pool pool, ProgressReporter progressReporter) {
@@ -58,7 +57,7 @@ public void testEnded(TestIdentifier test, Map<String, String> testMetrics) {
 
     @Override
     public void testRunFailed(String errorMessage) {
-        poolProgressTracker.failedTest();
+        poolProgressTracker.failedTestRun();
     }
 
     @Override

diff --git a/fork-runner/src/test/java/com/shazam/fork/runner/FakeProgressReporter.java b/fork-runner/src/test/java/com/shazam/fork/runner/FakeProgressReporter.java
@@ -64,7 +64,12 @@ public long millisSinceTestsStarted() {
     }
 
     @Override
-    public int getFailures() {
+    public int getTestFailures() {
+        return 0;
+    }
+
+    @Override
+    public int getTestRunFailures() {
         return 0;
     }