feat(cassettes): Store more information about executed checks in cass…

…ettes

docs/changelog.rst

@@ -11,6 +11,7 @@ Changelog
 - Shortcut for calling & validation. `#738`_
 - New hook to pre-commit, rstcheck, as well as updates to documentation based on rstcheck `#734`_
 - New check for maximum response time and corresponding CLI option ``--max-response-time`` `#716`_
+- New field with information about executed checks in cassettes. `#702`_
 
 **Fixed**
 
@@ -1385,6 +1386,7 @@ Deprecated
 .. _#716: https://github.com/schemathesis/schemathesis/issues/716
 .. _#708: https://github.com/schemathesis/schemathesis/issues/708
 .. _#705: https://github.com/schemathesis/schemathesis/issues/705
+.. _#702: https://github.com/schemathesis/schemathesis/issues/702
 .. _#692: https://github.com/schemathesis/schemathesis/issues/692
 .. _#686: https://github.com/schemathesis/schemathesis/issues/686
 .. _#684: https://github.com/schemathesis/schemathesis/issues/684

docs/cli.rst

@@ -233,6 +233,10 @@ It might look like this:
       seed: '1'
       elapsed: '0.00123'
       recorded_at: '2020-04-22T17:52:51.275318'
+      checks:
+        - name: 'not_a_server_error'
+          status: 'FAILURE'
+          message: 'Received a response with 5xx status code: 500'
       request:
         uri: 'http://127.0.0.1/api/failure'
         method: 'GET'
@@ -259,6 +263,7 @@ Schemathesis provides the following extra fields:
 - ``http_interactions.status``. Type of test outcome is one of ``SUCCESS``, ``FAILURE``, ``ERROR``.
 - ``http_interactions.seed``. The Hypothesis seed used in that particular case could be used as an argument to ``--hypothesis-seed`` CLI option to reproduce this request.
 - ``http_interactions.elapsed``. Time in seconds that a request took.
+- ``http_interactions.checks``. A list of executed checks and and their status.
 
 To work with the cassette, you could use `yq <https://github.com/mikefarah/yq>`_ or any similar tool.
 Show response body content of first failed interaction:

src/schemathesis/cli/cassettes.py

@@ -14,6 +14,7 @@
 from .. import constants
 from ..models import Interaction
 from ..runner import events
+from ..runner.serialization import SerializedCheck
 from .context import ExecutionContext
 from .handlers import EventHandler
 
@@ -45,7 +46,14 @@ def handle_event(self, context: ExecutionContext, event: events.ExecutionEvent)
             # Seed is always present at this point, the original Optional[int] type is there because `TestResult`
             # instance is created before `seed` is generated on the hypothesis side
             seed = cast(int, event.result.seed)
-            self.queue.put(Process(status=event.status.name.upper(), seed=seed, interactions=event.result.interactions))
+            self.queue.put(
+                Process(
+                    status=event.status.name.upper(),
+                    seed=seed,
+                    interactions=event.result.interactions,
+                    checks=event.result.checks,
+                )
+            )
         if isinstance(event, events.Finished):
             self.shutdown()
 
@@ -69,6 +77,7 @@ class Process:
     status: str = attr.ib()  # pragma: no mutate
     seed: int = attr.ib()  # pragma: no mutate
     interactions: List[Interaction] = attr.ib()  # pragma: no mutate
+    checks: List[SerializedCheck] = attr.ib()  # pragma: no mutate
 
 
 @attr.s(slots=True)  # pragma: no mutate
@@ -104,6 +113,15 @@ def format_header_values(values: List[str]) -> str:
     def format_headers(headers: Dict[str, List[str]]) -> str:
         return "\n".join(f"      {name}:\n{format_header_values(values)}" for name, values in headers.items())
 
+    def format_check_message(message: Optional[str]) -> str:
+        return "~" if message is None else f"'{message}'"
+
+    def format_checks(checks: List[SerializedCheck]) -> str:
+        return "\n".join(
+            f"    - name: '{check.name}'\n      status: '{check.value.name.upper()}'\n      message: {format_check_message(check.message)}"
+            for check in checks
+        )
+
     while True:
         item = queue.get()
         if isinstance(item, Initialize):
@@ -120,6 +138,8 @@ def format_headers(headers: Dict[str, List[str]]) -> str:
   seed: '{item.seed}'
   elapsed: '{interaction.response.elapsed}'
   recorded_at: '{interaction.recorded_at}'
+  checks:
+{format_checks(item.checks)}
   request:
     uri: '{interaction.request.uri}'
     method: '{interaction.request.method}'

test/cli/test_cassettes.py

@@ -37,6 +37,11 @@ def test_store_cassette(cli, schema_url, cassette_path):
     assert float(cassette["http_interactions"][0]["elapsed"]) >= 0
     data = base64.b64decode(cassette["http_interactions"][0]["response"]["body"]["base64_string"])
     assert data == b'{"success": true}'
+    assert all("checks" in interaction for interaction in cassette["http_interactions"])
+    assert len(cassette["http_interactions"][0]["checks"]) == 1
+    assert cassette["http_interactions"][0]["checks"][0]["name"] == "not_a_server_error"
+    assert cassette["http_interactions"][0]["checks"][0]["status"] == "SUCCESS"
+    assert cassette["http_interactions"][0]["checks"][0]["message"] is None
 
 
 def test_encoding_error(testdir, cli, cassette_path, openapi3_base_url):