feat: A code snippet to reproduce a failed check when running Python …

…tests

Ref: #793

docs/changelog.rst

@@ -14,6 +14,7 @@ Changelog
 - New ``response_headers_conformance`` check that verifies the presence of all headers defined for a response. `#742`_
 - New field with information about executed checks in cassettes. `#702`_
 - New ``port`` parameter added to ``from_uri()`` method. `#706`_
+- A code snippet to reproduce a failed check when running Python tests. `#793`_
 - Python 3.9 support. `#731`_
 
 **Fixed**
@@ -1383,6 +1384,7 @@ Deprecated
 .. _0.3.0: https://github.com/schemathesis/schemathesis/compare/v0.2.0...v0.3.0
 .. _0.2.0: https://github.com/schemathesis/schemathesis/compare/v0.1.0...v0.2.0
 
+.. _#793: https://github.com/schemathesis/schemathesis/issues/793
 .. _#788: https://github.com/schemathesis/schemathesis/issues/788
 .. _#783: https://github.com/schemathesis/schemathesis/issues/783
 .. _#768: https://github.com/schemathesis/schemathesis/issues/768

src/schemathesis/models.py

@@ -17,7 +17,7 @@
 from starlette.testclient import TestClient as ASGIClient
 
 from .constants import USER_AGENT
-from .exceptions import InvalidSchema
+from .exceptions import CheckFailed, InvalidSchema
 from .types import Body, Cookies, FormData, Headers, PathParameters, Query
 from .utils import GenericResponse, WSGIResponse
 
@@ -263,10 +263,14 @@ def validate_response(
         for check in checks:
             try:
                 check(response, self)
-            except AssertionError as exc:
+            except CheckFailed as exc:
                 errors.append(exc.args[0])
         if errors:
-            raise AssertionError(*errors)
+            formatted_errors = "\n\n".join(f"{idx}. {error}" for idx, error in enumerate(errors, 1))
+            raise CheckFailed(
+                f"\n\n{formatted_errors}\n\n----------\n\n"
+                f"Run this Python code to reproduce this response: \n\n    {self.get_code_to_reproduce()}"
+            )
 
     def call_and_validate(
         self,

test/test_models.py

@@ -189,7 +189,19 @@ def test_(case):
     try:
         case.validate_response(response)
     except AssertionError as exc:
-        assert exc.args[0] == "Received a response with a status code, which is not defined in the schema: 418\n\nDeclared status codes: 200"
+        assert exc.args[0].split("\n") == [
+          '',
+          '',
+          '1. Received a response with a status code, which is not defined in the schema: 418',
+          '',
+          'Declared status codes: 200',
+          '',
+          '----------',
+          '',
+          'Run this Python code to reproduce this response: ',
+          '',
+          "    requests.get('http://localhost/v1/users', headers={'User-Agent': 'schemathesis/2.5.1'})"
+    ]
 """
     )
     result = testdir.runpytest()

test/test_pytest.py

@@ -189,3 +189,30 @@ def test(case):
     # Then the test should fail instead of error
     result = testdir.runpytest()
     result.assert_outcomes(failed=1)
+
+
+def test_failure_reproduction_message(testdir, openapi3_base_url):
+    # When a test fails
+    testdir.make_test(
+        f"""
+schema.base_url = "{openapi3_base_url}"
+
+@schema.parametrize(endpoint="failure")
+def test(case):
+    response = case.call()
+    case.validate_response(response)
+    """,
+        paths={"/failure": {"get": {"responses": {"200": {"description": "OK"}}}}},
+    )
+    # Then there should be a helpful message in the output
+    result = testdir.runpytest()
+    result.assert_outcomes(failed=1)
+    result.stdout.re_match_lines(
+        [
+            r".+1. Received a response with 5xx status code: 500",
+            r".+2. Received a response with a status code, which is not defined in the schema: 500",
+            r".+Declared status codes: 200",
+            r".+Run this Python code to reproduce this response:",
+            rf".+requests.get\('{openapi3_base_url}/failure', headers={{'User-Agent': 'schemathesis/2.5.1'}}\)",
+        ]
+    )