feat(checks): Add custom check for max response time

docs/changelog.rst

@@ -10,6 +10,7 @@ Changelog
 - Support PEP561 `#748`_
 - Shortcut for calling & validation. `#738`_
 - New hook to pre-commit, rstcheck, as well as updates to documentation based on rstcheck `#734`_
+- New check for maximum response time and corresponding CLI option ``--max-response-time`` `#716`_
 
 `2.5.1`_ - 2020-09-30
 ---------------------
@@ -1371,6 +1372,7 @@ Deprecated
 .. _#721: https://github.com/schemathesis/schemathesis/issues/721
 .. _#719: https://github.com/schemathesis/schemathesis/issues/719
 .. _#718: https://github.com/schemathesis/schemathesis/issues/718
+.. _#716: https://github.com/schemathesis/schemathesis/issues/716
 .. _#708: https://github.com/schemathesis/schemathesis/issues/708
 .. _#705: https://github.com/schemathesis/schemathesis/issues/705
 .. _#692: https://github.com/schemathesis/schemathesis/issues/692

src/schemathesis/cli/__init__.py

@@ -92,6 +92,11 @@ def schemathesis(pre_run: Optional[str] = None) -> None:
 @click.option(
     "--checks", "-c", multiple=True, help="List of checks to run.", type=CHECKS_TYPE, default=DEFAULT_CHECKS_NAMES
 )
+@click.option(
+    "--max-response-time",
+    help="A custom check that will fail if the response time is greater than the specified one in milliseconds.",
+    type=click.IntRange(min=1),
+)
 @click.option(
     "--target",
     "-t",
@@ -247,6 +252,7 @@ def run(  # pylint: disable=too-many-arguments
     auth_type: str,
     headers: Dict[str, str],
     checks: Iterable[str] = DEFAULT_CHECKS_NAMES,
+    max_response_time: Optional[int] = None,
     targets: Iterable[str] = DEFAULT_TARGETS_NAMES,
     exit_first: bool = False,
     endpoints: Optional[Filter] = None,
@@ -302,6 +308,7 @@ def run(  # pylint: disable=too-many-arguments
         exit_first=exit_first,
         store_interactions=store_network_log is not None,
         checks=selected_checks,
+        max_response_time=max_response_time,
         targets=selected_targets,
         workers_num=workers_num,
         validate_schema=validate_schema,

src/schemathesis/runner/__init__.py

@@ -31,6 +31,7 @@ def prepare(  # pylint: disable=too-many-arguments
     *,
     # Runtime behavior
     checks: Iterable[CheckFunction] = DEFAULT_CHECKS,
+    max_response_time: Optional[int] = None,
     targets: Iterable[Target] = DEFAULT_TARGETS,
     workers_num: int = 1,
     seed: Optional[int] = None,
@@ -89,6 +90,7 @@ def prepare(  # pylint: disable=too-many-arguments
         app=app,
         validate_schema=validate_schema,
         checks=checks,
+        max_response_time=max_response_time,
         targets=targets,
         hypothesis_options=hypothesis_options,
         seed=seed,
@@ -136,6 +138,7 @@ def execute_from_schema(
     app: Optional[str] = None,
     validate_schema: bool = True,
     checks: Iterable[CheckFunction],
+    max_response_time: Optional[int] = None,
     targets: Iterable[Target],
     workers_num: int = 1,
     hypothesis_options: Dict[str, Any],
@@ -185,6 +188,7 @@ def execute_from_schema(
                 runner = ThreadPoolRunner(
                     schema=schema,
                     checks=checks,
+                    max_response_time=max_response_time,
                     targets=targets,
                     hypothesis_settings=hypothesis_options,
                     auth=auth,
@@ -202,6 +206,7 @@ def execute_from_schema(
                 runner = ThreadPoolASGIRunner(
                     schema=schema,
                     checks=checks,
+                    max_response_time=max_response_time,
                     targets=targets,
                     hypothesis_settings=hypothesis_options,
                     auth=auth,
@@ -218,6 +223,7 @@ def execute_from_schema(
                 runner = ThreadPoolWSGIRunner(
                     schema=schema,
                     checks=checks,
+                    max_response_time=max_response_time,
                     targets=targets,
                     hypothesis_settings=hypothesis_options,
                     auth=auth,
@@ -236,6 +242,7 @@ def execute_from_schema(
                 runner = SingleThreadRunner(
                     schema=schema,
                     checks=checks,
+                    max_response_time=max_response_time,
                     targets=targets,
                     hypothesis_settings=hypothesis_options,
                     auth=auth,
@@ -252,6 +259,7 @@ def execute_from_schema(
                 runner = SingleThreadASGIRunner(
                     schema=schema,
                     checks=checks,
+                    max_response_time=max_response_time,
                     targets=targets,
                     hypothesis_settings=hypothesis_options,
                     auth=auth,
@@ -267,6 +275,7 @@ def execute_from_schema(
                 runner = SingleThreadWSGIRunner(
                     schema=schema,
                     checks=checks,
+                    max_response_time=max_response_time,
                     targets=targets,
                     hypothesis_settings=hypothesis_options,
                     auth=auth,

src/schemathesis/runner/impl/core.py

@@ -32,6 +32,7 @@ def get_hypothesis_settings(hypothesis_options: Dict[str, Any]) -> hypothesis.se
 class BaseRunner:
     schema: BaseSchema = attr.ib()  # pragma: no mutate
     checks: Iterable[CheckFunction] = attr.ib()  # pragma: no mutate
+    max_response_time: Optional[int] = attr.ib()  # pragma: no mutate
     targets: Iterable[Target] = attr.ib()  # pragma: no mutate
     hypothesis_settings: hypothesis.settings = attr.ib(converter=get_hypothesis_settings)  # pragma: no mutate
     auth: Optional[RawAuth] = attr.ib(default=None)  # pragma: no mutate
@@ -171,7 +172,14 @@ def reraise(error: AssertionError) -> InvalidSchema:
         return exc
 
 
-def run_checks(case: Case, checks: Iterable[CheckFunction], result: TestResult, response: GenericResponse) -> None:
+def run_checks(  # pylint: disable=too-many-arguments
+    case: Case,
+    checks: Iterable[CheckFunction],
+    result: TestResult,
+    response: GenericResponse,
+    elapsed_time: float,
+    max_response_time: Optional[int] = None,
+) -> None:
     errors = []
 
     for check in checks:
@@ -188,6 +196,14 @@ def run_checks(case: Case, checks: Iterable[CheckFunction], result: TestResult,
             errors.append(exc)
             result.add_failure(check_name, case, message)
 
+    if max_response_time:
+        if elapsed_time > max_response_time:
+            message = "Check 'max_response_time' failed"
+            errors.append(AssertionError(message))
+            result.add_failure("max_response_time", case, message)
+        else:
+            result.add_success("max_response_time", case)
+
     if errors:
         raise get_grouped_exception(*errors)
 
@@ -217,15 +233,38 @@ def network_test(
     store_interactions: bool,
     headers: Optional[Dict[str, Any]],
     feedback: Feedback,
+    max_response_time: Optional[int],
 ) -> None:
     """A single test body will be executed against the target."""
     # pylint: disable=too-many-arguments
     headers = headers or {}
     headers.setdefault("User-Agent", USER_AGENT)
     timeout = prepare_timeout(request_timeout)
-    response = _network_test(case, checks, targets, result, session, timeout, store_interactions, headers, feedback)
+    response = _network_test(
+        case,
+        checks,
+        targets,
+        result,
+        session,
+        timeout,
+        store_interactions,
+        headers,
+        feedback,
+        max_response_time,
+    )
     add_cases(
-        case, response, _network_test, checks, targets, result, session, timeout, store_interactions, headers, feedback
+        case,
+        response,
+        _network_test,
+        checks,
+        targets,
+        result,
+        session,
+        timeout,
+        store_interactions,
+        headers,
+        feedback,
+        max_response_time,
     )
 
 
@@ -239,14 +278,15 @@ def _network_test(
     store_interactions: bool,
     headers: Optional[Dict[str, Any]],
     feedback: Feedback,
+    max_response_time: Optional[int],
 ) -> requests.Response:
     # pylint: disable=too-many-arguments
     response = case.call(session=session, headers=headers, timeout=timeout)
     context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
     if store_interactions:
         result.store_requests_response(response)
-    run_checks(case, checks, result, response)
+    run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
     feedback.add_test_case(case, response)
     return response
 
@@ -277,11 +317,14 @@ def wsgi_test(
     headers: Optional[Dict[str, Any]],
     store_interactions: bool,
     feedback: Feedback,
+    max_response_time: Optional[int],
 ) -> None:
     # pylint: disable=too-many-arguments
     headers = _prepare_wsgi_headers(headers, auth, auth_type)
-    response = _wsgi_test(case, checks, targets, result, headers, store_interactions, feedback)
-    add_cases(case, response, _wsgi_test, checks, targets, result, headers, store_interactions, feedback)
+    response = _wsgi_test(case, checks, targets, result, headers, store_interactions, feedback, max_response_time)
+    add_cases(
+        case, response, _wsgi_test, checks, targets, result, headers, store_interactions, feedback, max_response_time
+    )
 
 
 def _wsgi_test(
@@ -292,6 +335,7 @@ def _wsgi_test(
     headers: Dict[str, Any],
     store_interactions: bool,
     feedback: Feedback,
+    max_response_time: Optional[int],
 ) -> WSGIResponse:
     # pylint: disable=too-many-arguments
     with catching_logs(LogCaptureHandler(), level=logging.DEBUG) as recorded:
@@ -303,7 +347,7 @@ def _wsgi_test(
     if store_interactions:
         result.store_wsgi_response(case, response, headers, elapsed)
     result.logs.extend(recorded.records)
-    run_checks(case, checks, result, response)
+    run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
     feedback.add_test_case(case, response)
     return response
 
@@ -335,13 +379,16 @@ def asgi_test(
     store_interactions: bool,
     headers: Optional[Dict[str, Any]],
     feedback: Feedback,
+    max_response_time: Optional[int],
 ) -> None:
     """A single test body will be executed against the target."""
     # pylint: disable=too-many-arguments
     headers = headers or {}
 
-    response = _asgi_test(case, checks, targets, result, store_interactions, headers, feedback)
-    add_cases(case, response, _asgi_test, checks, targets, result, store_interactions, headers, feedback)
+    response = _asgi_test(case, checks, targets, result, store_interactions, headers, feedback, max_response_time)
+    add_cases(
+        case, response, _asgi_test, checks, targets, result, store_interactions, headers, feedback, max_response_time
+    )
 
 
 def _asgi_test(
@@ -352,13 +399,14 @@ def _asgi_test(
     store_interactions: bool,
     headers: Optional[Dict[str, Any]],
     feedback: Feedback,
+    max_response_time: Optional[int],
 ) -> requests.Response:
     # pylint: disable=too-many-arguments
     response = case.call_asgi(headers=headers)
     context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
     if store_interactions:
         result.store_requests_response(response)
-    run_checks(case, checks, result, response)
+    run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
     feedback.add_test_case(case, response)
     return response

src/schemathesis/runner/impl/solo.py

@@ -22,6 +22,7 @@ def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, N
                 self.hypothesis_settings,
                 self.seed,
                 checks=self.checks,
+                max_response_time=self.max_response_time,
                 targets=self.targets,
                 results=results,
                 session=session,
@@ -40,6 +41,7 @@ def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, N
             self.hypothesis_settings,
             self.seed,
             checks=self.checks,
+            max_response_time=self.max_response_time,
             targets=self.targets,
             results=results,
             auth=self.auth,
@@ -58,6 +60,7 @@ def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, N
             self.hypothesis_settings,
             self.seed,
             checks=self.checks,
+            max_response_time=self.max_response_time,
             targets=self.targets,
             results=results,
             headers=self.headers,

src/schemathesis/runner/impl/threadpool.py

@@ -230,7 +230,11 @@ def _get_worker_kwargs(self, tasks_queue: Queue, events_queue: Queue, results: T
             "results": results,
             "stateful": self.stateful,
             "stateful_recursion_limit": self.stateful_recursion_limit,
-            "kwargs": {"request_timeout": self.request_timeout, "store_interactions": self.store_interactions},
+            "kwargs": {
+                "request_timeout": self.request_timeout,
+                "store_interactions": self.store_interactions,
+                "max_response_time": self.max_response_time,
+            },
         }
 
 
@@ -254,6 +258,7 @@ def _get_worker_kwargs(self, tasks_queue: Queue, events_queue: Queue, results: T
                 "auth_type": self.auth_type,
                 "headers": self.headers,
                 "store_interactions": self.store_interactions,
+                "max_response_time": self.max_response_time,
             },
         }
 
@@ -274,7 +279,10 @@ def _get_worker_kwargs(self, tasks_queue: Queue, events_queue: Queue, results: T
             "results": results,
             "stateful": self.stateful,
             "stateful_recursion_limit": self.stateful_recursion_limit,
-            "kwargs": {"store_interactions": self.store_interactions},
+            "kwargs": {
+                "store_interactions": self.store_interactions,
+                "max_response_time": self.max_response_time,
+            },
         }
 
 

test/cli/test_commands.py

@@ -135,6 +135,10 @@ def test_commands_version(cli):
             "Error: Invalid value for '--header' / '-H': Header value should be latin-1 encodable",
         ),
         (("run", "//test"), "Error: Invalid SCHEMA, must be a valid URL or file path."),
+        (
+            ("run", "http://127.0.0.1", "--max-response-time=0"),
+            "Error: Invalid value for '--max-response-time': 0 is smaller than the minimum valid value 1.",
+        ),
     ),
 )
 def test_commands_run_errors(cli, args, error):
@@ -162,6 +166,11 @@ def test_commands_run_help(cli):
         "  -c, --checks [not_a_server_error|status_code_conformance|"
         "content_type_conformance|response_schema_conformance|all]",
         "                                  List of checks to run.",
+        "  --max-response-time INTEGER RANGE",
+        "                                  A custom check that will fail if the response",
+        "                                  time is greater than the specified one in",
+        "                                  milliseconds.",
+        "",
         "  -t, --target [response_time|all]",
         "                                  Targets for input generation.",
         "  -x, --exitfirst                 Exit instantly on first error or failed test.",
@@ -264,6 +273,7 @@ def test_commands_run_help(cli):
             },
         ),
         (["--hypothesis-deadline=None"], {"hypothesis_options": {"deadline": None}}),
+        (["--max-response-time=10"], {"max_response_time": 10}),
     ),
 )
 def test_execute_arguments(cli, mocker, simple_schema, args, expected):
@@ -299,6 +309,7 @@ def test_execute_arguments(cli, mocker, simple_schema, args, expected):
         "request_timeout": None,
         "store_interactions": False,
         "seed": None,
+        "max_response_time": None,
         **expected,
     }
 
@@ -1464,3 +1475,21 @@ def test_get_request_with_body(testdir, cli, base_url, schema_with_get_payload):
     assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout
     lines = result.stdout.splitlines()
     assert "schemathesis.exceptions.InvalidSchema: Body parameters are defined for GET request." in lines
+
+
+@pytest.mark.endpoints("slow")
+@pytest.mark.parametrize("workers", (1, 2))
+def test_max_response_time(cli, server, schema_url, workers):
+    # When maximum response time check is specified in the CLI and the request takes more time
+    result = cli.run(schema_url, "--max-response-time=50", f"--workers={workers}")
+    # Then the whole Schemathesis run should fail
+    assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout
+    # And the given endpoint should be displayed as a failure
+    lines = result.stdout.split("\n")
+    if workers == 1:
+        assert lines[10].startswith("GET /api/slow F")
+    else:
+        assert lines[10].startswith("F")
+    # And the proper error message should be displayed
+    assert "max_response_time                     0 / 2 passed          FAILED" in result.stdout
+    assert "Check 'max_response_time' failed" in result.stdout