fix: Status of individual interactions in VCR cassettes

Before this change, all statuses were taken from the overall test outcome,
  rather than from the check results for a particular response.

Ref: #695

docs/changelog.rst

@@ -16,6 +16,8 @@ Changelog
 **Fixed**
 
 - Default ``User-Agent`` header in ``Case.call``. `#717`_
+- Status of individual interactions in VCR cassettes. Before this change, all statuses were taken from the overall test outcome,
+  rather than from the check results for a particular response. `#695`_
 
 **Changed**
 
@@ -1387,6 +1389,7 @@ Deprecated
 .. _#708: https://github.com/schemathesis/schemathesis/issues/708
 .. _#705: https://github.com/schemathesis/schemathesis/issues/705
 .. _#702: https://github.com/schemathesis/schemathesis/issues/702
+.. _#695: https://github.com/schemathesis/schemathesis/issues/695
 .. _#692: https://github.com/schemathesis/schemathesis/issues/692
 .. _#686: https://github.com/schemathesis/schemathesis/issues/686
 .. _#684: https://github.com/schemathesis/schemathesis/issues/684

docs/cli.rst

@@ -260,7 +260,7 @@ Schemathesis provides the following extra fields:
 
 - ``command``. Full CLI command used to run Schemathesis.
 - ``http_interactions.id``. A numeric interaction ID within the current cassette.
-- ``http_interactions.status``. Type of test outcome is one of ``SUCCESS``, ``FAILURE``, ``ERROR``.
+- ``http_interactions.status``. Type of test outcome is one of ``SUCCESS``, ``FAILURE``. The status value is calculated from individual checks statuses - if any check failed, then the final status is ``FAILURE``.
 - ``http_interactions.seed``. The Hypothesis seed used in that particular case could be used as an argument to ``--hypothesis-seed`` CLI option to reproduce this request.
 - ``http_interactions.elapsed``. Time in seconds that a request took.
 - ``http_interactions.checks``. A list of executed checks and and their status.
@@ -288,7 +288,7 @@ Saved cassettes can be replayed with ``schemathesis replay`` command. Additional
 replay by these parameters:
 
 - ``id``. Specific, unique ID;
-- ``status``. Replay only interactions with this status (``SUCCESS``, ``FAILURE`` or ``ERROR``);
+- ``status``. Replay only interactions with this status (``SUCCESS`` or ``FAILURE``);
 - ``uri``. A regular expression for request URI;
 - ``method``. A regular expression for request method;
 

src/schemathesis/cli/cassettes.py

@@ -48,7 +48,6 @@ def handle_event(self, context: ExecutionContext, event: events.ExecutionEvent)
             seed = cast(int, event.result.seed)
             self.queue.put(
                 Process(
-                    status=event.status.name.upper(),
                     seed=seed,
                     interactions=event.result.interactions,
                     checks=event.result.checks,
@@ -74,7 +73,6 @@ class Initialize:
 class Process:
     """A new chunk of data should be processed."""
 
-    status: str = attr.ib()  # pragma: no mutate
     seed: int = attr.ib()  # pragma: no mutate
     interactions: List[Interaction] = attr.ib()  # pragma: no mutate
     checks: List[SerializedCheck] = attr.ib()  # pragma: no mutate
@@ -132,9 +130,10 @@ def format_checks(checks: List[SerializedCheck]) -> str:
             )
         elif isinstance(item, Process):
             for interaction in item.interactions:
+                status = interaction.status.name.upper()
                 stream.write(
                     f"""\n- id: '{current_id}'
-  status: '{item.status}'
+  status: '{status}'
   seed: '{item.seed}'
   elapsed: '{interaction.response.elapsed}'
   recorded_at: '{interaction.recorded_at}'

src/schemathesis/models.py

@@ -542,17 +542,26 @@ class Interaction:
 
     request: Request = attr.ib()  # pragma: no mutate
     response: Response = attr.ib()  # pragma: no mutate
+    status: Status = attr.ib()  # pragma: no mutate
     recorded_at: str = attr.ib(factory=lambda: datetime.datetime.now().isoformat())  # pragma: no mutate
 
     @classmethod
-    def from_requests(cls, response: requests.Response) -> "Interaction":
-        return cls(request=Request.from_prepared_request(response.request), response=Response.from_requests(response))
+    def from_requests(cls, response: requests.Response, status: Status) -> "Interaction":
+        return cls(
+            request=Request.from_prepared_request(response.request),
+            response=Response.from_requests(response),
+            status=status,
+        )
 
     @classmethod
-    def from_wsgi(cls, case: Case, response: WSGIResponse, headers: Dict[str, Any], elapsed: float) -> "Interaction":
+    def from_wsgi(  # pylint: disable=too-many-arguments
+        cls, case: Case, response: WSGIResponse, headers: Dict[str, Any], elapsed: float, status: Status
+    ) -> "Interaction":
         session = requests.Session()
         session.headers.update(headers)
-        return cls(request=Request.from_case(case, session), response=Response.from_wsgi(response, elapsed))
+        return cls(
+            request=Request.from_case(case, session), response=Response.from_wsgi(response, elapsed), status=status
+        )
 
 
 @attr.s(slots=True, repr=False)  # pragma: no mutate
@@ -593,11 +602,13 @@ def add_failure(self, name: str, example: Case, message: str) -> None:
     def add_error(self, exception: Exception, example: Optional[Case] = None) -> None:
         self.errors.append((exception, example))
 
-    def store_requests_response(self, response: requests.Response) -> None:
-        self.interactions.append(Interaction.from_requests(response))
+    def store_requests_response(self, response: requests.Response, status: Status) -> None:
+        self.interactions.append(Interaction.from_requests(response, status))
 
-    def store_wsgi_response(self, case: Case, response: WSGIResponse, headers: Dict[str, Any], elapsed: float) -> None:
-        self.interactions.append(Interaction.from_wsgi(case, response, headers, elapsed))
+    def store_wsgi_response(  # pylint: disable=too-many-arguments
+        self, case: Case, response: WSGIResponse, headers: Dict[str, Any], elapsed: float, status: Status
+    ) -> None:
+        self.interactions.append(Interaction.from_wsgi(case, response, headers, elapsed, status))
 
 
 @attr.s(slots=True, repr=False)  # pragma: no mutate

src/schemathesis/runner/impl/core.py

@@ -284,9 +284,15 @@ def _network_test(
     response = case.call(session=session, headers=headers, timeout=timeout)
     context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
-    if store_interactions:
-        result.store_requests_response(response)
-    run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
+    status = Status.success
+    try:
+        run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
+    except CheckFailed:
+        status = Status.failure
+        raise
+    finally:
+        if store_interactions:
+            result.store_requests_response(response, status)
     feedback.add_test_case(case, response)
     return response
 
@@ -344,10 +350,16 @@ def _wsgi_test(
         elapsed = time.monotonic() - start
     context = TargetContext(case=case, response=response, response_time=elapsed)
     run_targets(targets, context)
-    if store_interactions:
-        result.store_wsgi_response(case, response, headers, elapsed)
     result.logs.extend(recorded.records)
-    run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
+    status = Status.success
+    try:
+        run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
+    except CheckFailed:
+        status = Status.failure
+        raise
+    finally:
+        if store_interactions:
+            result.store_wsgi_response(case, response, headers, elapsed, status)
     feedback.add_test_case(case, response)
     return response
 
@@ -405,8 +417,14 @@ def _asgi_test(
     response = case.call_asgi(headers=headers)
     context = TargetContext(case=case, response=response, response_time=response.elapsed.total_seconds())
     run_targets(targets, context)
-    if store_interactions:
-        result.store_requests_response(response)
-    run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
+    status = Status.success
+    try:
+        run_checks(case, checks, result, response, context.response_time * 1000, max_response_time)
+    except CheckFailed:
+        status = Status.failure
+        raise
+    finally:
+        if store_interactions:
+            result.store_requests_response(response, status)
     feedback.add_test_case(case, response)
     return response

test/cli/test_cassettes.py

@@ -22,6 +22,10 @@ def load_cassette(path):
         return yaml.safe_load(fd)
 
 
+def load_response_body(cassette, idx):
+    return base64.b64decode(cassette["http_interactions"][idx]["response"]["body"]["base64_string"])
+
+
 @pytest.mark.endpoints("success", "upload_file")
 def test_store_cassette(cli, schema_url, cassette_path):
     result = cli.run(
@@ -35,15 +39,37 @@ def test_store_cassette(cli, schema_url, cassette_path):
     assert cassette["http_interactions"][0]["status"] == "SUCCESS"
     assert cassette["http_interactions"][0]["seed"] == "1"
     assert float(cassette["http_interactions"][0]["elapsed"]) >= 0
-    data = base64.b64decode(cassette["http_interactions"][0]["response"]["body"]["base64_string"])
-    assert data == b'{"success": true}'
+    assert load_response_body(cassette, 0) == b'{"success": true}'
     assert all("checks" in interaction for interaction in cassette["http_interactions"])
     assert len(cassette["http_interactions"][0]["checks"]) == 1
     assert cassette["http_interactions"][0]["checks"][0]["name"] == "not_a_server_error"
     assert cassette["http_interactions"][0]["checks"][0]["status"] == "SUCCESS"
     assert cassette["http_interactions"][0]["checks"][0]["message"] is None
 
 
+@pytest.mark.endpoints("flaky")
+def test_interaction_status(cli, openapi3_schema_url, cassette_path):
+    # See GH-695
+    # When an endpoint has responses with SUCCESS and FAILURE statuses
+    result = cli.run(
+        openapi3_schema_url,
+        f"--store-network-log={cassette_path}",
+        "--hypothesis-max-examples=5",
+        "--hypothesis-seed=1",
+    )
+    assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout
+    cassette = load_cassette(cassette_path)
+    assert len(cassette["http_interactions"]) == 3
+    # Then their statuses should be reflected in the "status" field
+    # And it should not be overridden by the overall test status
+    assert cassette["http_interactions"][0]["status"] == "FAILURE"
+    assert load_response_body(cassette, 0) == b"500: Internal Server Error"
+    assert cassette["http_interactions"][1]["status"] == "SUCCESS"
+    assert load_response_body(cassette, 1) == b'{"result": "flaky!"}'
+    assert cassette["http_interactions"][2]["status"] == "SUCCESS"
+    assert load_response_body(cassette, 2) == b'{"result": "flaky!"}'
+
+
 def test_encoding_error(testdir, cli, cassette_path, openapi3_base_url):
     # See GH-708
     # When the schema expects an input that is not ascii and represented as UTF-8