[Core, JUnit, Android] Add the ambiguous result type.

Report ambiguous steps as a separate result type, instead of reporting them as failed. The new ambiguous result type will make the exit code non-zero (the same as the failed result type currently used when ambiguous step matches are found).
cucumber · Jul 7, 2017 · aa1e5a4 · aa1e5a4
1 parent b249304
commit aa1e5a4
Show file tree

Hide file tree

Showing 18 changed files with 99 additions and 28 deletions.
diff --git a/android/src/main/java/cucumber/runtime/android/AndroidInstrumentationReporter.java b/android/src/main/java/cucumber/runtime/android/AndroidInstrumentationReporter.java
@@ -194,6 +194,10 @@ void finishTestCase() {
                     instrumentation.sendStatus(StatusCodes.ERROR, testResult);
                 }
                 break;
+            case AMBIGUOUS:
+                testResult.putString(StatusKeys.STACK, getStackTrace(severestResult.getError()));
+                instrumentation.sendStatus(StatusCodes.ERROR, testResult);
+                break;
             case PENDING:
                 testResult.putString(StatusKeys.STACK, severestResult.getErrorMessage());
                 instrumentation.sendStatus(StatusCodes.ERROR, testResult);
@@ -214,7 +218,7 @@ void finishTestCase() {
     /**
      * Creates a template bundle for reporting the start and end of a test.
      *
-     * @param path of the feature file of the current execution
+     * @param path         of the feature file of the current execution
      * @param testCaseName of the test case of the current execution
      * @return the new {@link Bundle}
      */

diff --git a/core/src/main/java/cucumber/api/Result.java b/core/src/main/java/cucumber/api/Result.java
@@ -15,6 +15,7 @@ public static enum Type {
         SKIPPED,
         PENDING,
         UNDEFINED,
+        AMBIGUOUS,
         FAILED;
 
         public static Type fromLowerCaseName(String lowerCaseName) {

diff --git a/core/src/main/java/cucumber/api/TestStep.java b/core/src/main/java/cucumber/api/TestStep.java
@@ -3,6 +3,7 @@
 import cucumber.api.event.TestStepFinished;
 import cucumber.api.event.TestStepStarted;
 import cucumber.runner.EventBus;
+import cucumber.runtime.AmbiguousStepDefinitionsException;
 import cucumber.runtime.DefinitionMatch;
 import cucumber.runtime.UndefinedStepDefinitionException;
 import gherkin.pickles.Argument;
@@ -92,6 +93,9 @@ private Result.Type mapThrowableToStatus(Throwable t) {
         if (t.getClass() == UndefinedStepDefinitionException.class) {
             return Result.Type.UNDEFINED;
         }
+        if (t.getClass() == AmbiguousStepDefinitionsException.class) {
+            return Result.Type.AMBIGUOUS;
+        }
         return Result.Type.FAILED;
     }
 

diff --git a/core/src/main/java/cucumber/runner/Runner.java b/core/src/main/java/cucumber/runner/Runner.java
@@ -121,7 +121,7 @@ private void addTestStepsForPickleSteps(List<TestStep> testSteps, PickleEvent pi
                     match = new UndefinedStepDefinitionMatch(step);
                 }
             } catch (AmbiguousStepDefinitionsException e) {
-                match = new AmbiguousStepDefinitionsMatch(step, e);
+                match = new AmbiguousStepDefinitionsMatch(pickleEvent.uri, step, e);
             } catch (Throwable t) {
                 match = new FailedStepInstantiationMatch(pickleEvent.uri, step, t);
             }

diff --git a/core/src/main/java/cucumber/runtime/AmbiguousStepDefinitionsMatch.java b/core/src/main/java/cucumber/runtime/AmbiguousStepDefinitionsMatch.java
@@ -3,11 +3,13 @@
 import cucumber.api.Scenario;
 import gherkin.pickles.PickleStep;
 
+import java.util.Collections;
+
 public class AmbiguousStepDefinitionsMatch extends StepDefinitionMatch {
     private AmbiguousStepDefinitionsException exception;
 
-    public AmbiguousStepDefinitionsMatch(PickleStep step, AmbiguousStepDefinitionsException e) {
-        super(null, new NoStepDefinition(), null, step, null);
+    public AmbiguousStepDefinitionsMatch(String uri, PickleStep step, AmbiguousStepDefinitionsException e) {
+        super(Collections.<Argument>emptyList(), new NoStepDefinition(), uri, step, null);
         this.exception = e;
     }
 

diff --git a/core/src/main/java/cucumber/runtime/ScenarioImpl.java b/core/src/main/java/cucumber/runtime/ScenarioImpl.java
@@ -20,7 +20,7 @@
 import static java.util.Arrays.asList;
 
 public class ScenarioImpl implements Scenario {
-    private static final List<Result.Type> SEVERITY = asList(Result.Type.PASSED, Result.Type.SKIPPED, Result.Type.PENDING, Result.Type.UNDEFINED, Result.Type.FAILED);
+    private static final List<Result.Type> SEVERITY = asList(Result.Type.PASSED, Result.Type.SKIPPED, Result.Type.PENDING, Result.Type.UNDEFINED, Result.Type.AMBIGUOUS, Result.Type.FAILED);
     private final List<Result> stepResults = new ArrayList<Result>();
     private final List<PickleTag> tags;
     private final String uri;

diff --git a/core/src/main/java/cucumber/runtime/Stats.java b/core/src/main/java/cucumber/runtime/Stats.java
@@ -22,6 +22,7 @@ class Stats {
     private Formats formats;
     private Locale locale;
     private List<String> failedScenarios = new ArrayList<String>();
+    private List<String> ambiguousScenarios = new ArrayList<String>();
     private List<String> pendingScenarios = new ArrayList<String>();
     private List<String> undefinedScenarios = new ArrayList<String>();
 
@@ -67,6 +68,7 @@ private void printScenarioCounts(PrintStream out) {
     private void printSubCounts(PrintStream out, SubCounts subCounts) {
         boolean addComma = false;
         addComma = printSubCount(out, subCounts.failed, Result.Type.FAILED, addComma);
+        addComma = printSubCount(out, subCounts.ambiguous, Result.Type.AMBIGUOUS, addComma);
         addComma = printSubCount(out, subCounts.skipped, Result.Type.SKIPPED, addComma);
         addComma = printSubCount(out, subCounts.pending, Result.Type.PENDING, addComma);
         addComma = printSubCount(out, subCounts.undefined, Result.Type.UNDEFINED, addComma);
@@ -93,6 +95,7 @@ private void printDuration(PrintStream out) {
 
     private void printNonZeroResultScenarios(PrintStream out, boolean isStrict) {
         printScenarios(out, failedScenarios, Result.Type.FAILED);
+        printScenarios(out, ambiguousScenarios, Result.Type.AMBIGUOUS);
         if (isStrict) {
             printScenarios(out, pendingScenarios, Result.Type.PENDING);
             printScenarios(out, undefinedScenarios, Result.Type.UNDEFINED);
@@ -138,6 +141,9 @@ private void addResultToSubCount(SubCounts subCounts, Result.Type resultStatus)
         case FAILED:
             subCounts.failed++;
             break;
+        case AMBIGUOUS:
+            subCounts.ambiguous++;
+            break;
         case PENDING:
             subCounts.pending++;
             break;
@@ -158,6 +164,9 @@ public void addScenario(Result.Type resultStatus, String scenarioDesignation) {
         case FAILED:
             failedScenarios.add(scenarioDesignation);
             break;
+        case AMBIGUOUS:
+            ambiguousScenarios.add(scenarioDesignation);
+            break;
         case PENDING:
             pendingScenarios.add(scenarioDesignation);
             break;
@@ -172,12 +181,13 @@ public void addScenario(Result.Type resultStatus, String scenarioDesignation) {
     class SubCounts {
         public int passed = 0;
         public int failed = 0;
+        public int ambiguous = 0;
         public int skipped = 0;
         public int pending = 0;
         public int undefined = 0;
 
         public int getTotal() {
-            return passed + failed + skipped + pending + undefined;
+            return passed + failed + ambiguous + skipped + pending + undefined;
         }
     }
 }
diff --git a/core/src/main/java/cucumber/runtime/formatter/AnsiFormats.java b/core/src/main/java/cucumber/runtime/formatter/AnsiFormats.java
@@ -15,6 +15,8 @@ public class AnsiFormats implements Formats {
         put("executing_arg", new ColorFormat(AnsiEscapes.GREY, AnsiEscapes.INTENSITY_BOLD));
         put("failed", new ColorFormat(AnsiEscapes.RED));
         put("failed_arg", new ColorFormat(AnsiEscapes.RED, AnsiEscapes.INTENSITY_BOLD));
+        put("ambiguous", new ColorFormat(AnsiEscapes.RED));
+        put("ambiguous_arg", new ColorFormat(AnsiEscapes.RED, AnsiEscapes.INTENSITY_BOLD));
         put("passed", new ColorFormat(AnsiEscapes.GREEN));
         put("passed_arg", new ColorFormat(AnsiEscapes.GREEN, AnsiEscapes.INTENSITY_BOLD));
         put("outline", new ColorFormat(AnsiEscapes.CYAN));

diff --git a/core/src/main/java/cucumber/runtime/formatter/JUnitFormatter.java b/core/src/main/java/cucumber/runtime/formatter/JUnitFormatter.java
@@ -250,6 +250,9 @@ public void addTestCaseElement(Document doc, Element tc, Result result) {
             if (result.is(Result.Type.FAILED)) {
                 addStackTrace(sb, result);
                 child = createElementWithMessage(doc, sb, "failure", result.getErrorMessage());
+            } else if (result.is(Result.Type.AMBIGUOUS)) {
+                addStackTrace(sb, result);
+                child = createElementWithMessage(doc, sb, "failure", result.getErrorMessage());
             } else if (result.is(Result.Type.PENDING) || result.is(Result.Type.UNDEFINED)) {
                 if (treatConditionallySkippedAsFailure) {
                     child = createElementWithMessage(doc, sb, "failure", "The scenario has pending or undefined step(s)");

diff --git a/core/src/main/java/cucumber/runtime/formatter/ProgressFormatter.java b/core/src/main/java/cucumber/runtime/formatter/ProgressFormatter.java
@@ -21,13 +21,15 @@ class ProgressFormatter implements Formatter, ColorAware {
         put(Result.Type.PENDING, 'P');
         put(Result.Type.SKIPPED, '-');
         put(Result.Type.FAILED, 'F');
+        put(Result.Type.AMBIGUOUS, 'A');
     }};
     private static final Map<Result.Type, AnsiEscapes> ANSI_ESCAPES = new HashMap<Result.Type, AnsiEscapes>() {{
         put(Result.Type.PASSED, AnsiEscapes.GREEN);
         put(Result.Type.UNDEFINED, AnsiEscapes.YELLOW);
         put(Result.Type.PENDING, AnsiEscapes.YELLOW);
         put(Result.Type.SKIPPED, AnsiEscapes.CYAN);
         put(Result.Type.FAILED, AnsiEscapes.RED);
+        put(Result.Type.AMBIGUOUS, AnsiEscapes.RED);
     }};
 
     private final NiceAppendable out;

diff --git a/core/src/main/java/cucumber/runtime/formatter/TestNGFormatter.java b/core/src/main/java/cucumber/runtime/formatter/TestNGFormatter.java
@@ -226,7 +226,7 @@ public void finish(Document doc, Element element) {
             Result skipped = null;
             Result failed = null;
             for (Result result : results) {
-                if (result.is(Result.Type.FAILED)) {
+                if (result.is(Result.Type.FAILED) || result.is(Result.Type.AMBIGUOUS)) {
                     failed = result;
                 }
                 if (result.is(Result.Type.UNDEFINED) || result.is(Result.Type.PENDING)) {

diff --git a/core/src/test/java/cucumber/runtime/AmbiguousStepDefinitionMatchsTest.java b/core/src/test/java/cucumber/runtime/AmbiguousStepDefinitionMatchsTest.java
@@ -11,7 +11,7 @@
 public class AmbiguousStepDefinitionMatchsTest {
     public static final String ENGLISH = "en";
     public final AmbiguousStepDefinitionsException e = mock(AmbiguousStepDefinitionsException.class);
-    public final AmbiguousStepDefinitionsMatch match = new AmbiguousStepDefinitionsMatch(mock(PickleStep.class), e);
+    public final AmbiguousStepDefinitionsMatch match = new AmbiguousStepDefinitionsMatch("uri", mock(PickleStep.class), e);
 
     @Test
     public void throws_ambiguous_step_definitions_exception_when_run() {

diff --git a/core/src/test/java/cucumber/runtime/RuntimeTest.java b/core/src/test/java/cucumber/runtime/RuntimeTest.java
@@ -302,8 +302,8 @@ public void should_add_ambiguous_match_as_failed_result_to_the_summary_counter()
         runtime.printStats(new PrintStream(baos));
 
         assertThat(baos.toString(), containsString(String.format(""+
-                "1 Scenarios (1 failed)%n" +
-                "1 Steps (1 failed)%n")));
+                "1 Scenarios (1 ambiguous)%n" +
+                "1 Steps (1 ambiguous)%n")));
     }
 
     @Test

diff --git a/core/src/test/java/cucumber/runtime/StatsTest.java b/core/src/test/java/cucumber/runtime/StatsTest.java
@@ -48,43 +48,46 @@ public void should_only_print_sub_counts_if_not_zero() {
     }
 
     @Test
-    public void should_print_sub_counts_in_order_failed_skipped_pending_undefined_passed() {
+    public void should_print_sub_counts_in_order_failed_ambiguous_skipped_pending_undefined_passed() {
         Stats counter = createMonochromeSummaryCounter();
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
         addOneStepScenario(counter, Result.Type.PASSED);
         addOneStepScenario(counter, Result.Type.FAILED);
+        addOneStepScenario(counter, Result.Type.AMBIGUOUS);
         addOneStepScenario(counter, Result.Type.PENDING);
         addOneStepScenario(counter, Result.Type.UNDEFINED);
         addOneStepScenario(counter, Result.Type.SKIPPED);
         counter.printStats(new PrintStream(baos), isStrict(false));
 
         assertThat(baos.toString(), containsString(String.format("" +
-                "5 Scenarios (1 failed, 1 skipped, 1 pending, 1 undefined, 1 passed)%n" +
-                "5 Steps (1 failed, 1 skipped, 1 pending, 1 undefined, 1 passed)%n")));
+                "6 Scenarios (1 failed, 1 ambiguous, 1 skipped, 1 pending, 1 undefined, 1 passed)%n" +
+                "6 Steps (1 failed, 1 ambiguous, 1 skipped, 1 pending, 1 undefined, 1 passed)%n")));
     }
 
     @Test
-    public void should_print_sub_counts_in_order_failed_skipped_undefined_passed_in_color() {
+    public void should_print_sub_counts_in_order_failed_ambiguous_skipped_undefined_passed_in_color() {
         Stats counter = createColorSummaryCounter();
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
         addOneStepScenario(counter, Result.Type.PASSED);
         addOneStepScenario(counter, Result.Type.FAILED);
+        addOneStepScenario(counter, Result.Type.AMBIGUOUS);
         addOneStepScenario(counter, Result.Type.PENDING);
         addOneStepScenario(counter, Result.Type.UNDEFINED);
         addOneStepScenario(counter, Result.Type.SKIPPED);
         counter.printStats(new PrintStream(baos), isStrict(false));
 
-        String colorSubCounts =
+        String colorSubCounts = "" +
                 AnsiEscapes.RED + "1 failed" + AnsiEscapes.RESET + ", " +
+                AnsiEscapes.RED + "1 ambiguous" + AnsiEscapes.RESET + ", " +
                 AnsiEscapes.CYAN + "1 skipped" + AnsiEscapes.RESET + ", " +
                 AnsiEscapes.YELLOW + "1 pending" + AnsiEscapes.RESET + ", " +
                 AnsiEscapes.YELLOW + "1 undefined" + AnsiEscapes.RESET + ", " +
                 AnsiEscapes.GREEN + "1 passed" + AnsiEscapes.RESET;
         assertThat(baos.toString(), containsString(String.format("" +
-                "5 Scenarios (" + colorSubCounts + ")%n" +
-                "5 Steps (" + colorSubCounts + ")%n")));
+                "6 Scenarios (" + colorSubCounts + ")%n" +
+                "6 Steps (" + colorSubCounts + ")%n")));
     }
 
     @Test
@@ -155,12 +158,14 @@ public void should_use_locale_for_decimal_separator() {
     }
 
     @Test
-    public void should_print_failed_scenarios() {
+    public void should_print_failed_ambiguous_scenarios() {
         Stats counter = createMonochromeSummaryCounter();
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
         counter.addStep(createResultWithStatus(Result.Type.FAILED));
         counter.addScenario(Result.Type.FAILED, "path/file.feature:3 # Scenario: scenario_name");
+        counter.addStep(createResultWithStatus(Result.Type.AMBIGUOUS));
+        counter.addScenario(Result.Type.AMBIGUOUS, "path/file.feature:3 # Scenario: scenario_name");
         counter.addStep(createResultWithStatus(Result.Type.UNDEFINED));
         counter.addScenario(Result.Type.UNDEFINED, "path/file.feature:3 # Scenario: scenario_name");
         counter.addStep(createResultWithStatus(Result.Type.PENDING));
@@ -171,16 +176,21 @@ public void should_print_failed_scenarios() {
                 "Failed scenarios:%n" +
                 "path/file.feature:3 # Scenario: scenario_name%n" +
                 "%n" +
-                "3 Scenarios")));
+                "Ambiguous scenarios:%n" +
+                "path/file.feature:3 # Scenario: scenario_name%n" +
+                "%n" +
+                "4 Scenarios")));
     }
 
     @Test
-    public void should_print_failed_pending_undefined_scenarios_if_strict() {
+    public void should_print_failed_ambiguous_pending_undefined_scenarios_if_strict() {
         Stats counter = createMonochromeSummaryCounter();
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
         counter.addStep(createResultWithStatus(Result.Type.FAILED));
         counter.addScenario(Result.Type.FAILED, "path/file.feature:3 # Scenario: scenario_name");
+        counter.addStep(createResultWithStatus(Result.Type.AMBIGUOUS));
+        counter.addScenario(Result.Type.AMBIGUOUS, "path/file.feature:3 # Scenario: scenario_name");
         counter.addStep(createResultWithStatus(Result.Type.UNDEFINED));
         counter.addScenario(Result.Type.UNDEFINED, "path/file.feature:3 # Scenario: scenario_name");
         counter.addStep(createResultWithStatus(Result.Type.PENDING));
@@ -191,13 +201,16 @@ public void should_print_failed_pending_undefined_scenarios_if_strict() {
                 "Failed scenarios:%n" +
                 "path/file.feature:3 # Scenario: scenario_name%n" +
                 "%n" +
+                "Ambiguous scenarios:%n" +
+                "path/file.feature:3 # Scenario: scenario_name%n" +
+                "%n" +
                 "Pending scenarios:%n" +
                 "path/file.feature:3 # Scenario: scenario_name%n" +
                 "%n" +
                 "Undefined scenarios:%n" +
                 "path/file.feature:3 # Scenario: scenario_name%n" +
                 "%n" +
-                "3 Scenarios")));
+                "4 Scenarios")));
     }
 
     private void addOneStepScenario(Stats counter, Result.Type status) {

diff --git a/core/src/test/java/cucumber/runtime/TestHelper.java b/core/src/test/java/cucumber/runtime/TestHelper.java
@@ -69,6 +69,8 @@ public static Result result(Result.Type status) {
         switch (status) {
         case FAILED:
             return result(status, mockAssertionFailedError());
+        case AMBIGUOUS:
+            return result(status, mockAmbiguousStepDefinitionException());
         case PENDING:
             return result(status, new PendingException());
         default:
@@ -251,6 +253,20 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
         return error;
     }
 
+    private static AmbiguousStepDefinitionsException mockAmbiguousStepDefinitionException() {
+        AmbiguousStepDefinitionsException exception = mock(AmbiguousStepDefinitionsException.class);
+        Answer<Object> printStackTraceHandler = new Answer<Object>() {
+            @Override
+            public Object answer(InvocationOnMock invocation) throws Throwable {
+                PrintWriter writer = (PrintWriter) invocation.getArguments()[0];
+                writer.print("the stack trace");
+                return null;
+            }
+        };
+        doAnswer(printStackTraceHandler).when(exception).printStackTrace((PrintWriter) any());
+        return exception;
+    }
+
     public static SimpleEntry<String, Result> hookEntry(String type, Result result) {
         return new SimpleEntry<String, Result>(type, result);
     }

diff --git a/junit/src/main/java/cucumber/runtime/junit/JUnitReporter.java b/junit/src/main/java/cucumber/runtime/junit/JUnitReporter.java
@@ -127,9 +127,11 @@ void handleStepResult(TestStep testStep, Result result) {
             stepErrors.add(new UndefinedThrowable(testStep.getStepText()));
             addFailureOrFailedAssumptionDependingOnStrictMode(stepNotifier, error);
             break;
+        case AMBIGUOUS:
         case FAILED:
             stepErrors.add(error);
             stepNotifier.addFailure(error);
+            break;
         }
         stepNotifier.fireTestFinished();
     }
@@ -159,10 +161,12 @@ void handleTestCaseResult(Result result) {
                 addFailureOrFailedAssumptionDependingOnStrictMode(pickleRunnerNotifier, error);
             }
             break;
+        case AMBIGUOUS:
         case FAILED:
             for (Throwable error : stepErrors) {
                 pickleRunnerNotifier.addFailure(error);
             }
+            break;
         }
         pickleRunnerNotifier.fireTestFinished();
     }