feat(cli): CLI option to fail after the first error

docs/changelog.rst

@@ -6,6 +6,11 @@ Changelog
 `Unreleased`_
 -------------
 
+Added
+~~~~~
+
+- ``-x``/``--exitfirst`` CLI option to exit after first failed test. `#378`_
+
 `0.23.6`_ - 2020-01-28
 ----------------------
 
@@ -642,6 +647,7 @@ Fixed
 .. _0.3.0: https://github.com/kiwicom/schemathesis/compare/v0.2.0...v0.3.0
 .. _0.2.0: https://github.com/kiwicom/schemathesis/compare/v0.1.0...v0.2.0
 
+.. _#378: https://github.com/kiwicom/schemathesis/issues/378
 .. _#376: https://github.com/kiwicom/schemathesis/issues/376
 .. _#374: https://github.com/kiwicom/schemathesis/issues/374
 .. _#371: https://github.com/kiwicom/schemathesis/issues/371

src/schemathesis/cli/__init__.py

@@ -49,6 +49,9 @@ def schemathesis(pre_run: Optional[str] = None) -> None:
 @click.option(
     "--checks", "-c", multiple=True, help="List of checks to run.", type=CHECKS_TYPE, default=DEFAULT_CHECKS_NAMES
 )
+@click.option(
+    "-x", "--exitfirst", "exit_first", is_flag=True, default=False, help="Exit instantly on first error or failed test."
+)
 @click.option(
     "--auth", "-a", help="Server user and password. Example: USER:PASSWORD", type=str, callback=callbacks.validate_auth
 )
@@ -129,6 +132,7 @@ def run(  # pylint: disable=too-many-arguments
     auth_type: str,
     headers: Dict[str, str],
     checks: Iterable[str] = DEFAULT_CHECKS_NAMES,
+    exit_first: bool = False,
     endpoints: Optional[Filter] = None,
     methods: Optional[Filter] = None,
     tags: Optional[Filter] = None,
@@ -173,6 +177,7 @@ def run(  # pylint: disable=too-many-arguments
             verbosity=hypothesis_verbosity,
         ),
         seed=hypothesis_seed,
+        exit_first=exit_first,
     )
     if validate_schema is False:
         options.setdefault("loader_options", {})["validate_schema"] = validate_schema

src/schemathesis/runner/__init__.py

@@ -48,6 +48,7 @@ class BaseRunner:
     headers: Optional[Dict[str, Any]] = attr.ib(default=None)
     request_timeout: Optional[int] = attr.ib(default=None)
     seed: Optional[int] = attr.ib(default=None)
+    exit_first: bool = attr.ib(default=False)
 
     def execute(self,) -> Generator[events.ExecutionEvent, None, None]:
         """Common logic for all runners."""
@@ -58,7 +59,14 @@ def execute(self,) -> Generator[events.ExecutionEvent, None, None]:
         )
         yield initialized
 
-        yield from self._execute(results)
+        for event in self._execute(results):
+            if (
+                self.exit_first
+                and isinstance(event, events.AfterExecution)
+                and event.status in (Status.error, Status.failure)
+            ):
+                break
+            yield event
 
         yield events.Finished(results=results, schema=self.schema, running_time=time.time() - initialized.start_time)
 
@@ -282,6 +290,7 @@ def execute_from_schema(
     headers: Optional[Dict[str, Any]] = None,
     request_timeout: Optional[int] = None,
     seed: Optional[int] = None,
+    exit_first: bool = False,
 ) -> Generator[events.ExecutionEvent, None, None]:
     """Execute tests for the given schema.
 
@@ -299,6 +308,7 @@ def execute_from_schema(
                 headers=headers,
                 seed=seed,
                 workers_num=workers_num,
+                exit_first=exit_first,
             )
         else:
             runner = ThreadPoolRunner(
@@ -310,6 +320,7 @@ def execute_from_schema(
                 headers=headers,
                 seed=seed,
                 request_timeout=request_timeout,
+                exit_first=exit_first,
             )
     else:
         if schema.app:
@@ -321,6 +332,7 @@ def execute_from_schema(
                 auth_type=auth_type,
                 headers=headers,
                 seed=seed,
+                exit_first=exit_first,
             )
         else:
             runner = SingleThreadRunner(
@@ -332,6 +344,7 @@ def execute_from_schema(
                 headers=headers,
                 seed=seed,
                 request_timeout=request_timeout,
+                exit_first=exit_first,
             )
 
     yield from runner.execute()
@@ -425,6 +438,7 @@ def prepare(  # pylint: disable=too-many-arguments
     hypothesis_options: Optional[Dict[str, Any]] = None,
     loader: Callable = from_uri,
     seed: Optional[int] = None,
+    exit_first: bool = False,
 ) -> Generator[events.ExecutionEvent, None, None]:
     """Prepare a generator that will run test cases against the given API definition."""
     api_options = api_options or {}
@@ -434,7 +448,13 @@ def prepare(  # pylint: disable=too-many-arguments
         loader_options["base_url"] = get_base_url(schema_uri)
     schema = loader(schema_uri, **loader_options)
     return execute_from_schema(
-        schema, checks, hypothesis_options=hypothesis_options, seed=seed, workers_num=workers_num, **api_options
+        schema,
+        checks,
+        hypothesis_options=hypothesis_options,
+        seed=seed,
+        workers_num=workers_num,
+        exit_first=exit_first,
+        **api_options,
     )
 
 

test/cli/test_commands.py

@@ -115,6 +115,7 @@ def test_commands_run_help(cli):
         "  -c, --checks [not_a_server_error|status_code_conformance|"
         "content_type_conformance|response_schema_conformance|all]",
         "                                  List of checks to run.",
+        "  -x, --exitfirst                 Exit instantly on first error or failed test.",
         "  -a, --auth TEXT                 Server user and password. Example:",
         "                                  USER:PASSWORD",
         "  -A, --auth-type [basic|digest]  The authentication mechanism to be used.",
@@ -163,6 +164,7 @@ def test_commands_run_help(cli):
     (
         ([SCHEMA_URI], {"checks": DEFAULT_CHECKS, "workers_num": 1}),
         ([SCHEMA_URI, "--checks=all"], {"checks": ALL_CHECKS, "workers_num": 1}),
+        ([SCHEMA_URI, "--exitfirst"], {"checks": DEFAULT_CHECKS, "exit_first": True, "workers_num": 1}),
         (
             [SIMPLE_PATH, "--base-url=http://127.0.0.1"],
             {

test/runner/test_runner.py

@@ -375,3 +375,12 @@ def test_get_requests_auth():
 def test_get_wsgi_auth():
     with pytest.raises(ValueError, match="Digest auth is not supported for WSGI apps"):
         get_wsgi_auth(("test", "test"), "digest")
+
+
+@pytest.mark.endpoints("failure", "multiple_failures")
+def test_exit_first(args):
+    app, kwargs = args
+    results = prepare(**kwargs, exit_first=True)
+    results = list(results)
+    assert results[-1].results.has_failures is True
+    assert results[-1].results.failed_count == 1