fix: Reference resolving during response schema conformance check

docs/changelog.rst

@@ -26,6 +26,7 @@ Fixed
 ~~~~~
 
 - Add missing ``validate_schema`` argument to ``loaders.from_pytest_fixture``.
+- Reference resolving during response schema conformance check. `#539`_
 
 `1.3.4`_ - 2020-04-30
 ---------------------
@@ -1003,6 +1004,7 @@ Fixed
 .. _0.2.0: https://github.com/kiwicom/schemathesis/compare/v0.1.0...v0.2.0
 
 .. _#542: https://github.com/kiwicom/schemathesis/issues/542
+.. _#539: https://github.com/kiwicom/schemathesis/issues/539
 .. _#537: https://github.com/kiwicom/schemathesis/issues/537
 .. _#529: https://github.com/kiwicom/schemathesis/issues/529
 .. _#521: https://github.com/kiwicom/schemathesis/issues/521

src/schemathesis/checks.py

@@ -22,7 +22,7 @@ def not_a_server_error(response: GenericResponse, case: "Case") -> None:
 
 
 def status_code_conformance(response: GenericResponse, case: "Case") -> None:
-    responses = case.endpoint.definition.get("responses", {})
+    responses = case.endpoint.definition.raw.get("responses", {})
     # "default" can be used as the default response object for all HTTP codes that are not covered individually
     if "default" in responses:
         return
@@ -71,7 +71,7 @@ def response_schema_conformance(response: GenericResponse, case: "Case") -> None
     if not content_type.startswith("application/json"):
         return
     # the keys should be strings
-    responses = {str(key): value for key, value in case.endpoint.definition.get("responses", {}).items()}
+    responses = {str(key): value for key, value in case.endpoint.definition.raw.get("responses", {}).items()}
     status_code = str(response.status_code)
     if status_code in responses:
         definition = responses[status_code]
@@ -80,7 +80,7 @@ def response_schema_conformance(response: GenericResponse, case: "Case") -> None
     else:
         # No response defined for the received response status code
         return
-    schema = case.endpoint.schema._get_response_schema(definition)
+    schema = case.endpoint.schema._get_response_schema(definition, case.endpoint.definition.scope)
     if not schema:
         return
     if isinstance(response, requests.Response):

src/schemathesis/converter.py

@@ -1,8 +1,6 @@
 from copy import deepcopy
 from typing import Any, Dict
 
-from .utils import traverse_schema
-
 
 def to_json_schema(schema: Dict[str, Any], nullable_name: str) -> Dict[str, Any]:
     """Convert Open API parameters to JSON Schema.
@@ -25,12 +23,3 @@ def to_json_schema(schema: Dict[str, Any], nullable_name: str) -> Dict[str, Any]
         schema["type"] = "string"
         schema["format"] = "binary"
     return schema
-
-
-def to_json_schema_recursive(schema: Dict[str, Any], nullable_name: str) -> Dict[str, Any]:
-    """Apply ``to_json_schema`` recursively.
-
-    This version is needed for cases where the input schema was not resolved and ``to_json_schema`` wasn't applied
-    recursively.
-    """
-    return traverse_schema(schema, to_json_schema, nullable_name)

src/schemathesis/models.py

@@ -217,13 +217,26 @@ def empty_object() -> Dict[str, Any]:
     return {"properties": {}, "additionalProperties": False, "type": "object", "required": []}
 
 
+@attr.s(slots=True)  # pragma: no mutate
+class EndpointDefinition:
+    """A wrapper to store not resolved endpoint definitions.
+
+    To prevent recursion errors we need to store definitions without resolving references. But endpoint definitions
+    itself can be behind a reference (when there is a ``$ref`` in ``paths`` values), therefore we need to store this
+    scope change to have a proper reference resolving later.
+    """
+
+    raw: Dict[str, Any] = attr.ib()  # pragma: no mutate
+    scope: Optional[str] = attr.ib()  # pragma: no mutate
+
+
 @attr.s(slots=True)  # pragma: no mutate
 class Endpoint:
     """A container that could be used for test cases generation."""
 
     path: str = attr.ib()  # pragma: no mutate
     method: str = attr.ib()  # pragma: no mutate
-    definition: Dict[str, Any] = attr.ib()  # pragma: no mutate
+    definition: EndpointDefinition = attr.ib()  # pragma: no mutate
     schema: "BaseSchema" = attr.ib()  # pragma: no mutate
     app: Any = attr.ib(default=None)  # pragma: no mutate
     base_url: Optional[str] = attr.ib(default=None)  # pragma: no mutate

src/schemathesis/schemas.py

@@ -22,11 +22,11 @@
 from requests.structures import CaseInsensitiveDict
 
 from ._hypothesis import make_test_or_exception
-from .converter import to_json_schema, to_json_schema_recursive
+from .converter import to_json_schema
 from .exceptions import InvalidSchema
 from .filters import should_skip_by_tag, should_skip_endpoint, should_skip_method
 from .hooks import HookContext, HookDispatcher, HookLocation, HookScope, dispatch, warn_deprecated_hook
-from .models import Endpoint, empty_object
+from .models import Endpoint, EndpointDefinition, empty_object
 from .types import Filter, GenericTest, Hook, NotSet
 from .utils import NOT_SET, GenericResponse, StringDatesYAMLLoader, deprecated
 
@@ -161,7 +161,7 @@ def clone(  # pylint: disable=too-many-arguments
             validate_schema=validate_schema,  # type: ignore
         )
 
-    def _get_response_schema(self, definition: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    def _get_response_schema(self, definition: Dict[str, Any], scope: Optional[str]) -> Optional[Dict[str, Any]]:
         """Extract response schema from `responses`."""
         raise NotImplementedError
 
@@ -239,9 +239,13 @@ def get_all_endpoints(self) -> Generator[Endpoint, None, None]:
                 if should_skip_endpoint(full_path, self.endpoint):
                     continue
                 self.dispatch_hook("before_process_path", context, path, methods)
-                # Only one level is resolved for `raw_methods` so method names are available, but everything deeper
-                # is not resolved.
-                raw_methods = self.resolve(deepcopy(methods), RECURSION_DEPTH_LIMIT)
+                # We need to know a proper scope in what methods are.
+                # It will allow us to provide a proper reference resolving in `response_schema_conformance` and avoid
+                # recursion errors
+                if "$ref" in methods:
+                    scope, raw_methods = deepcopy(self.resolver.resolve(methods["$ref"]))
+                else:
+                    raw_methods, scope = deepcopy(methods), None
                 methods = self.resolve(methods)
                 common_parameters = get_common_parameters(methods)
                 for method, resolved_definition in methods.items():
@@ -255,7 +259,7 @@ def get_all_endpoints(self) -> Generator[Endpoint, None, None]:
                     parameters = itertools.chain(resolved_definition.get("parameters", ()), common_parameters)
                     # To prevent recursion errors we need to pass not resolved schema as well
                     # It could be used for response validation
-                    raw_definition = raw_methods[method]
+                    raw_definition = EndpointDefinition(raw_methods[method], scope)
                     yield self.make_endpoint(full_path, method, parameters, resolved_definition, raw_definition)
         except (KeyError, AttributeError, jsonschema.exceptions.RefResolutionError):
             raise InvalidSchema("Schema parsing failed. Please check your schema.")
@@ -266,7 +270,7 @@ def make_endpoint(  # pylint: disable=too-many-arguments
         method: str,
         parameters: Iterator[Dict[str, Any]],
         resolved_definition: Dict[str, Any],
-        raw_definition: Dict[str, Any],
+        raw_definition: EndpointDefinition,
     ) -> Endpoint:
         """Create JSON schemas for query, body, etc from Swagger parameters definitions."""
         base_url = self.base_url
@@ -370,18 +374,25 @@ def resolve(self, item: Union[Dict[str, Any], List], recursion_level: int = 0) -
                 item[idx] = self.resolve(sub_item, recursion_level)
         return item
 
+    def resolve_in_scope(self, item: Dict[str, Any], scope: Optional[str], recursion_level: int = 0) -> Dict[str, Any]:
+        if scope is not None:
+            with self.resolver.in_scope(scope):
+                return self.resolve(item, recursion_level)
+        return self.resolve(item, recursion_level)
+
     def prepare(self, item: Dict[str, Any]) -> Dict[str, Any]:
         """Parse schema extension, e.g. "x-nullable" field."""
         return to_json_schema(item, self.nullable_name)
 
-    def _get_response_schema(self, definition: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    def _get_response_schema(self, definition: Dict[str, Any], scope: Optional[str]) -> Optional[Dict[str, Any]]:
+        definition = self.resolve_in_scope(deepcopy(definition), scope, RECURSION_DEPTH_LIMIT)
         schema = definition.get("schema")
         if not schema:
             return None
-        return to_json_schema_recursive(schema, self.nullable_name)
+        return to_json_schema(schema, self.nullable_name)
 
     def get_content_types(self, endpoint: Endpoint, response: GenericResponse) -> List[str]:
-        produces = endpoint.definition.get("produces", None)
+        produces = endpoint.definition.raw.get("produces", None)
         if produces:
             return produces
         return self.raw_schema.get("produces", [])
@@ -421,7 +432,7 @@ def make_endpoint(  # pylint: disable=too-many-arguments
         method: str,
         parameters: Iterator[Dict[str, Any]],
         resolved_definition: Dict[str, Any],
-        raw_definition: Dict[str, Any],
+        raw_definition: EndpointDefinition,
     ) -> Endpoint:
         """Create JSON schemas for query, body, etc from Swagger parameters definitions."""
         endpoint = super().make_endpoint(full_path, method, parameters, resolved_definition, raw_definition)
@@ -462,16 +473,17 @@ def parameter_to_json_schema(self, data: Dict[str, Any]) -> Dict[str, Any]:
         # "schema" field is required for all parameters in Open API 3.0
         return super().parameter_to_json_schema(data["schema"])
 
-    def _get_response_schema(self, definition: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    def _get_response_schema(self, definition: Dict[str, Any], scope: Optional[str]) -> Optional[Dict[str, Any]]:
+        definition = self.resolve_in_scope(deepcopy(definition), scope, RECURSION_DEPTH_LIMIT)
         options = iter(definition.get("content", {}).values())
         option = next(options, None)
         if option:
-            return to_json_schema_recursive(option["schema"], self.nullable_name)
+            return to_json_schema(option["schema"], self.nullable_name)
         return None
 
     def get_content_types(self, endpoint: Endpoint, response: GenericResponse) -> List[str]:
         try:
-            responses = endpoint.definition["responses"]
+            responses = endpoint.definition.raw["responses"]
         except KeyError:
             # Possible to get if `validate_schema=False` is passed during schema creation
             raise InvalidSchema("Schema parsing failed. Please check your schema.")

test/conftest.py

@@ -1,6 +1,7 @@
 from textwrap import dedent
 
 import pytest
+import yaml
 from click.testing import CliRunner
 from hypothesis import settings
 
@@ -167,6 +168,79 @@ def fast_api_schema():
     }
 
 
+ROOT_SCHEMA = {
+    "openapi": "3.0.2",
+    "info": {"title": "Example API", "description": "An API to test Schemathesis", "version": "1.0.0"},
+    "paths": {"/teapot": {"$ref": "paths/teapot.yaml#/TeapotCreatePath"}},
+}
+TEAPOT_PATHS = {
+    "TeapotCreatePath": {
+        "post": {
+            "summary": "Test",
+            "requestBody": {
+                "description": "Test.",
+                "content": {
+                    "application/json": {"schema": {"$ref": "../schemas/teapot/create.yaml#/TeapotCreateRequest"}}
+                },
+                "required": True,
+            },
+            "responses": {"default": {"$ref": "../../common/responses.yaml#/DefaultError"}},
+            "tags": ["ancillaries"],
+        }
+    }
+}
+TEAPOT_CREATE_SCHEMAS = {
+    "TeapotCreateRequest": {
+        "type": "object",
+        "description": "Test",
+        "additionalProperties": False,
+        "properties": {"username": {"type": "string"}, "profile": {"$ref": "#/Profile"}},
+        "required": ["username", "profile"],
+    },
+    "Profile": {
+        "type": "object",
+        "description": "Test",
+        "additionalProperties": False,
+        "properties": {"id": {"type": "integer"}},
+        "required": ["id"],
+    },
+}
+COMMON_RESPONSES = {
+    "DefaultError": {
+        "description": "Probably an error",
+        "content": {
+            "application/json": {
+                "schema": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "properties": {"key": {"type": "string"}},
+                    "required": ["key"],
+                }
+            }
+        },
+    }
+}
+
+
+@pytest.fixture()
+def complex_schema(testdir):
+    # This schema includes:
+    #   - references to other files
+    #   - local references in referenced files
+    #   - different directories - relative paths to other files
+    schema_root = testdir.mkdir("root")
+    common = testdir.mkdir("common")
+    paths = schema_root.mkdir("paths")
+    schemas = schema_root.mkdir("schemas")
+    teapot_schemas = schemas.mkdir("teapot")
+    root = schema_root / "root.yaml"
+    root.write_text(yaml.dump(ROOT_SCHEMA), "utf8")
+    (paths / "teapot.yaml").write_text(yaml.dump(TEAPOT_PATHS), "utf8")
+    (teapot_schemas / "create.yaml").write_text(yaml.dump(TEAPOT_CREATE_SCHEMAS), "utf8")
+    (common / "responses.yaml").write_text(yaml.dump(COMMON_RESPONSES), "utf8")
+    return str(root)
+
+
 @pytest.fixture(scope="session")
 def swagger_20(simple_schema):
     return schemathesis.from_dict(simple_schema)

test/runner/test_checks.py

@@ -1,7 +1,9 @@
+import json
 from typing import Any, Dict
 
 import pytest
 import requests
+from hypothesis import given, settings
 
 import schemathesis
 from schemathesis import models
@@ -12,11 +14,12 @@
     status_code_conformance,
 )
 from schemathesis.exceptions import InvalidSchema
+from schemathesis.models import EndpointDefinition
 from schemathesis.schemas import BaseSchema
 
 
 def make_case(schema: BaseSchema, definition: Dict[str, Any]) -> models.Case:
-    endpoint = models.Endpoint("/path", "GET", definition=definition, schema=schema)
+    endpoint = models.Endpoint("/path", "GET", definition=EndpointDefinition(definition, None), schema=schema)
     return models.Case(endpoint)
 
 
@@ -311,3 +314,30 @@ def test_response_schema_conformance_invalid_openapi(openapi_30, content, defini
     case = make_case(openapi_30, definition)
     with pytest.raises(AssertionError):
         response_schema_conformance(response, case)
+
+
+@pytest.mark.hypothesis_nested
+def test_response_schema_conformance_references_invalid(complex_schema):
+    schema = schemathesis.from_path(complex_schema)
+
+    @given(case=schema.endpoints["/teapot"]["POST"].as_strategy())
+    @settings(max_examples=3)
+    def test(case):
+        response = make_response(json.dumps({"foo": 1}).encode())
+        with pytest.raises(AssertionError):
+            case.validate_response(response)
+
+    test()
+
+
+@pytest.mark.hypothesis_nested
+def test_response_schema_conformance_references_valid(complex_schema):
+    schema = schemathesis.from_path(complex_schema)
+
+    @given(case=schema.endpoints["/teapot"]["POST"].as_strategy())
+    @settings(max_examples=3)
+    def test(case):
+        response = make_response(json.dumps({"key": "foo"}).encode())
+        case.validate_response(response)
+
+    test()

test/test_converter.py

@@ -1,6 +1,7 @@
 import pytest
 
 from schemathesis import converter
+from schemathesis.utils import traverse_schema
 
 
 @pytest.mark.parametrize(
@@ -54,4 +55,4 @@ def test_to_jsonschema(schema, expected):
     ),
 )
 def test_to_jsonschema_recursive(schema, expected):
-    assert converter.to_json_schema_recursive(schema, "x-nullable") == expected
+    assert traverse_schema(schema, converter.to_json_schema, "x-nullable") == expected