runtime: don't fail the test run for undefined/ambiguous when in dry …

…run (#1814)

* update scenario (failing)

* clarify

* change logic

* refactor to share logic across serial+parallel

* add changelog

* add doco for dry run

* add link to changelog entry

CHANGELOG.md

@@ -34,6 +34,8 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO
 ### Removed
 
 ### Fixed
+
+* When running with `--dry-run`, undefined or ambiguous steps no longer cause the process to exit with code 1. ([#1814](https://github.com/cucumber/cucumber-js/pull/1814))
 * When running the help command, it now shows all available formatters under the --format option. 
   [#1798](https://github.com/cucumber/cucumber-js/pull/1798)
 

README.md

@@ -65,6 +65,7 @@ The following documentation is for master. See below the documentation for older
   * [Attachments](/docs/support_files/attachments.md)
   * [API Reference](/docs/support_files/api_reference.md)
 * Guides
+  * [Dry Run](./docs/dry_run.md)
   * [ES Modules](./docs/esm.md)
   * [Formatters](./docs/formatters.md)
   * [Running in parallel](./docs/parallel.md)

docs/dry_run.md

@@ -0,0 +1,20 @@
+# Dry Run
+
+You can run cucumber-js in "Dry Run" mode like this:
+
+```shell
+$ cucumber-js --dry-run
+```
+
+The effect is that cucumber-js will still do all the aggregation work of looking at your feature files, loading your support code etc but without actually executing the tests. Specifically:
+
+- No [hooks](./support_files/hooks.md) are executed
+- Steps are reported as "skipped" instead of being executed
+- Undefined and ambiguous steps are reported, but don't cause the process to fail
+
+A few examples where this is useful:
+
+- Finding unused step definitions with the [usage formatter](./formatters.md#usage)
+- Generating [snippets](./snippets.md) for all undefined steps with the [snippets formatter](./formatters.md#snippets)
+- Checking if your path, tag expression etc matches the scenarios you expect it to
+

features/dryrun_mode.feature

@@ -18,6 +18,7 @@ Feature: Dryrun mode
       Given('a step', function() {})
       """
     When I run cucumber-js with `--dry-run`
+    And it passes
     Then scenario "some scenario" step "Given a step" has status "skipped"
     And scenario "some scenario" has status "skipped"
 
@@ -30,12 +31,29 @@ Feature: Dryrun mode
       Given('a(n) step', function() {});
       """
     When I run cucumber-js with `--dry-run`
-    Then it fails
+    Then it passes
     And scenario "some scenario" step "Given a step" has status "ambiguous"
 
+  Scenario: pending step
+
+    Since steps aren't actually executed in dry run, a step that would resolve to pending
+    will still show up as skipped.
+
+    Given a file named "features/step_definitions/cucumber_steps.js" with:
+      """
+      const {Given} = require('@cucumber/cucumber')
+
+      Given('a step', function() {
+        return 'pending';
+      });
+      """
+    When I run cucumber-js with `--dry-run`
+    Then it passes
+    And scenario "some scenario" step "Given a step" has status "skipped"
+
   Scenario: undefined step
     When I run cucumber-js with `--dry-run`
-    Then it fails
+    Then it passes
     And scenario "some scenario" step "Given a step" has status "undefined"
 
   Scenario: hooks should not execute in dry run, serial runtime

src/runtime/helpers.ts

@@ -63,3 +63,21 @@ export function retriesForPickle(
   }
   return 0
 }
+
+export function shouldCauseFailure(
+  status: messages.TestStepResultStatus,
+  options: IRuntimeOptions
+): boolean {
+  if (options.dryRun) {
+    return false
+  }
+  const failureStatuses: messages.TestStepResultStatus[] = [
+    messages.TestStepResultStatus.AMBIGUOUS,
+    messages.TestStepResultStatus.FAILED,
+    messages.TestStepResultStatus.UNDEFINED,
+  ]
+  if (options.strict) {
+    failureStatuses.push(messages.TestStepResultStatus.PENDING)
+  }
+  return failureStatuses.includes(status)
+}

src/runtime/index.ts

@@ -2,7 +2,7 @@ import { EventDataCollector, formatLocation } from '../formatter/helpers'
 import StackTraceFilter from '../stack_trace_filter'
 import UserCodeRunner from '../user_code_runner'
 import VError from 'verror'
-import { retriesForPickle } from './helpers'
+import { retriesForPickle, shouldCauseFailure } from './helpers'
 import { IdGenerator } from '@cucumber/messages'
 import * as messages from '@cucumber/messages'
 import TestCaseRunner from './test_case_runner'
@@ -109,7 +109,7 @@ export default class Runtime {
       worldParameters: this.options.worldParameters,
     })
     const status = await testCaseRunner.run()
-    if (this.shouldCauseFailure(status)) {
+    if (shouldCauseFailure(status, this.options)) {
       this.success = false
     }
   }
@@ -157,15 +157,4 @@ export default class Runtime {
     }
     return this.success
   }
-
-  shouldCauseFailure(status: messages.TestStepResultStatus): boolean {
-    const failureStatuses: messages.TestStepResultStatus[] = [
-      messages.TestStepResultStatus.AMBIGUOUS,
-      messages.TestStepResultStatus.FAILED,
-      messages.TestStepResultStatus.UNDEFINED,
-    ]
-    if (this.options.strict)
-      failureStatuses.push(messages.TestStepResultStatus.PENDING)
-    return failureStatuses.includes(status)
-  }
 }

src/runtime/parallel/coordinator.ts

@@ -1,6 +1,6 @@
 import { ChildProcess, fork } from 'child_process'
 import path from 'path'
-import { retriesForPickle } from '../helpers'
+import { retriesForPickle, shouldCauseFailure } from '../helpers'
 import * as messages from '@cucumber/messages'
 import { EventEmitter } from 'events'
 import { EventDataCollector } from '../../formatter/helpers'
@@ -157,7 +157,7 @@ export default class Coordinator {
     )
     if (
       !testCaseFinished.willBeRetried &&
-      this.shouldCauseFailure(worstTestStepResult.status)
+      shouldCauseFailure(worstTestStepResult.status, this.options)
     ) {
       this.success = false
     }
@@ -214,11 +214,4 @@ export default class Coordinator {
     }
     worker.process.send(runCommand)
   }
-
-  shouldCauseFailure(status: messages.TestStepResultStatus): boolean {
-    return (
-      ['AMBIGUOUS', 'FAILED', 'UNDEFINED'].includes(status) ||
-      (status === 'PENDING' && this.options.strict)
-    )
-  }
 }