From 5e7d7b60cd7ac141904864bb8b55d972bc2b7cf9 Mon Sep 17 00:00:00 2001 From: Stanislav Tkachenko Date: Tue, 28 Jan 2020 17:05:37 +0100 Subject: [PATCH] feat(cli): CLI option to fail after the first error --- docs/changelog.rst | 6 ++++++ src/schemathesis/cli/__init__.py | 5 +++++ src/schemathesis/runner/__init__.py | 24 ++++++++++++++++++++++-- test/cli/test_commands.py | 2 ++ test/runner/test_runner.py | 9 +++++++++ 5 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b90c20f328..fd48f0e653 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,11 @@ Changelog `Unreleased`_ ------------- +Added +~~~~~ + +- ``-x``/``--exitfirst`` CLI option to exit after first failed test. `#378`_ + `0.23.6`_ - 2020-01-28 ---------------------- @@ -642,6 +647,7 @@ Fixed .. _0.3.0: https://github.com/kiwicom/schemathesis/compare/v0.2.0...v0.3.0 .. _0.2.0: https://github.com/kiwicom/schemathesis/compare/v0.1.0...v0.2.0 +.. _#378: https://github.com/kiwicom/schemathesis/issues/378 .. _#376: https://github.com/kiwicom/schemathesis/issues/376 .. _#374: https://github.com/kiwicom/schemathesis/issues/374 .. _#371: https://github.com/kiwicom/schemathesis/issues/371 diff --git a/src/schemathesis/cli/__init__.py b/src/schemathesis/cli/__init__.py index 6e76fa93f6..57ae0c4b01 100644 --- a/src/schemathesis/cli/__init__.py +++ b/src/schemathesis/cli/__init__.py @@ -49,6 +49,9 @@ def schemathesis(pre_run: Optional[str] = None) -> None: @click.option( "--checks", "-c", multiple=True, help="List of checks to run.", type=CHECKS_TYPE, default=DEFAULT_CHECKS_NAMES ) +@click.option( + "-x", "--exitfirst", "exit_first", is_flag=True, default=False, help="Exit instantly on first error or failed test." +) @click.option( "--auth", "-a", help="Server user and password. Example: USER:PASSWORD", type=str, callback=callbacks.validate_auth ) @@ -129,6 +132,7 @@ def run( # pylint: disable=too-many-arguments auth_type: str, headers: Dict[str, str], checks: Iterable[str] = DEFAULT_CHECKS_NAMES, + exit_first: bool = False, endpoints: Optional[Filter] = None, methods: Optional[Filter] = None, tags: Optional[Filter] = None, @@ -173,6 +177,7 @@ def run( # pylint: disable=too-many-arguments verbosity=hypothesis_verbosity, ), seed=hypothesis_seed, + exit_first=exit_first, ) if validate_schema is False: options.setdefault("loader_options", {})["validate_schema"] = validate_schema diff --git a/src/schemathesis/runner/__init__.py b/src/schemathesis/runner/__init__.py index 6e9a6343a5..852e2fd42f 100644 --- a/src/schemathesis/runner/__init__.py +++ b/src/schemathesis/runner/__init__.py @@ -48,6 +48,7 @@ class BaseRunner: headers: Optional[Dict[str, Any]] = attr.ib(default=None) request_timeout: Optional[int] = attr.ib(default=None) seed: Optional[int] = attr.ib(default=None) + exit_first: bool = attr.ib(default=False) def execute(self,) -> Generator[events.ExecutionEvent, None, None]: """Common logic for all runners.""" @@ -58,7 +59,14 @@ def execute(self,) -> Generator[events.ExecutionEvent, None, None]: ) yield initialized - yield from self._execute(results) + for event in self._execute(results): + if ( + self.exit_first + and isinstance(event, events.AfterExecution) + and event.status in (Status.error, Status.failure) + ): + break + yield event yield events.Finished(results=results, schema=self.schema, running_time=time.time() - initialized.start_time) @@ -282,6 +290,7 @@ def execute_from_schema( headers: Optional[Dict[str, Any]] = None, request_timeout: Optional[int] = None, seed: Optional[int] = None, + exit_first: bool = False, ) -> Generator[events.ExecutionEvent, None, None]: """Execute tests for the given schema. @@ -299,6 +308,7 @@ def execute_from_schema( headers=headers, seed=seed, workers_num=workers_num, + exit_first=exit_first, ) else: runner = ThreadPoolRunner( @@ -310,6 +320,7 @@ def execute_from_schema( headers=headers, seed=seed, request_timeout=request_timeout, + exit_first=exit_first, ) else: if schema.app: @@ -321,6 +332,7 @@ def execute_from_schema( auth_type=auth_type, headers=headers, seed=seed, + exit_first=exit_first, ) else: runner = SingleThreadRunner( @@ -332,6 +344,7 @@ def execute_from_schema( headers=headers, seed=seed, request_timeout=request_timeout, + exit_first=exit_first, ) yield from runner.execute() @@ -425,6 +438,7 @@ def prepare( # pylint: disable=too-many-arguments hypothesis_options: Optional[Dict[str, Any]] = None, loader: Callable = from_uri, seed: Optional[int] = None, + exit_first: bool = False, ) -> Generator[events.ExecutionEvent, None, None]: """Prepare a generator that will run test cases against the given API definition.""" api_options = api_options or {} @@ -434,7 +448,13 @@ def prepare( # pylint: disable=too-many-arguments loader_options["base_url"] = get_base_url(schema_uri) schema = loader(schema_uri, **loader_options) return execute_from_schema( - schema, checks, hypothesis_options=hypothesis_options, seed=seed, workers_num=workers_num, **api_options + schema, + checks, + hypothesis_options=hypothesis_options, + seed=seed, + workers_num=workers_num, + exit_first=exit_first, + **api_options, ) diff --git a/test/cli/test_commands.py b/test/cli/test_commands.py index bc9372f1cb..60443f599c 100644 --- a/test/cli/test_commands.py +++ b/test/cli/test_commands.py @@ -115,6 +115,7 @@ def test_commands_run_help(cli): " -c, --checks [not_a_server_error|status_code_conformance|" "content_type_conformance|response_schema_conformance|all]", " List of checks to run.", + " -x, --exitfirst Exit instantly on first error or failed test.", " -a, --auth TEXT Server user and password. Example:", " USER:PASSWORD", " -A, --auth-type [basic|digest] The authentication mechanism to be used.", @@ -163,6 +164,7 @@ def test_commands_run_help(cli): ( ([SCHEMA_URI], {"checks": DEFAULT_CHECKS, "workers_num": 1}), ([SCHEMA_URI, "--checks=all"], {"checks": ALL_CHECKS, "workers_num": 1}), + ([SCHEMA_URI, "--exitfirst"], {"checks": DEFAULT_CHECKS, "exit_first": True, "workers_num": 1}), ( [SIMPLE_PATH, "--base-url=http://127.0.0.1"], { diff --git a/test/runner/test_runner.py b/test/runner/test_runner.py index 0891789fba..872d22725d 100644 --- a/test/runner/test_runner.py +++ b/test/runner/test_runner.py @@ -375,3 +375,12 @@ def test_get_requests_auth(): def test_get_wsgi_auth(): with pytest.raises(ValueError, match="Digest auth is not supported for WSGI apps"): get_wsgi_auth(("test", "test"), "digest") + + +@pytest.mark.endpoints("failure", "multiple_failures") +def test_exit_first(args): + app, kwargs = args + results = prepare(**kwargs, exit_first=True) + results = list(results) + assert results[-1].results.has_failures is True + assert results[-1].results.failed_count == 1