chore: Improve reporting of jsonschema errors which are caused by n…

…on-string object keys

docs/changelog.rst

@@ -7,6 +7,7 @@ Changelog
 **Changed**
 
 - Show ``cURL`` code samples by default instead of Python. `#1269`_
+- Improve reporting of ``jsonschema`` errors which are caused by non-string object keys.
 
 `3.10.1`_ - 2021-10-04
 ----------------------

src/schemathesis/cli/output/default.py

@@ -408,7 +408,7 @@ def display_internal_error(context: ExecutionContext, event: events.InternalErro
             f"Error: {message}\n"
             f"Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks"
         )
-        if event.exception_type == "jsonschema.exceptions.ValidationError":
+        if event.exception_type == "schemathesis.exceptions.SchemaLoadingError":
             message += "\n" + DISABLE_SCHEMA_VALIDATION_MESSAGE
         click.secho(message, fg="red")
 

src/schemathesis/specs/openapi/loaders.py

@@ -1,5 +1,7 @@
+import io
 import pathlib
-from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Union
+from contextlib import suppress
+from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 from urllib.parse import urljoin
 
 import jsonschema
@@ -17,7 +19,7 @@
 from ...lazy import LazySchema
 from ...types import Filter, NotSet, PathLike
 from ...utils import NOT_SET, StringDatesYAMLLoader, WSGIResponse, require_relative_url, setup_headers
-from . import definitions
+from . import definitions, validation
 from .schemas import BaseOpenAPISchema, OpenApi30, SwaggerV20
 
 DataGenerationMethodInput = Union[DataGenerationMethod, Iterable[DataGenerationMethod]]
@@ -233,14 +235,45 @@ def _prepare_data_generation_methods(data_generation_methods: DataGenerationMeth
     return list(data_generation_methods)
 
 
+# It is a common case when API schemas are stored in the YAML format and HTTP status codes are numbers
+# The Open API spec requires HTTP status codes as strings
+DOC_ENTRY = "https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#patterned-fields-1"
+NUMERIC_STATUS_CODES_MESSAGE = f"""The input schema contains HTTP status codes as numbers.
+The Open API spec requires them to be strings:
+{DOC_ENTRY}
+Please, stringify the following status codes:"""
+NON_STRING_OBJECT_KEY = "The input schema contains non-string keys in sub-schemas"
+
+
+def _format_status_codes(status_codes: List[Tuple[int, List[Union[str, int]]]]) -> str:
+    buffer = io.StringIO()
+    for status_code, path in status_codes:
+        buffer.write(f" - {status_code} at schema['paths']")
+        for chunk in path:
+            buffer.write(f"[{repr(chunk)}]")
+        buffer.write("['responses']\n")
+    return buffer.getvalue().rstrip()
+
+
 def _maybe_validate_schema(
     instance: Dict[str, Any], validator: jsonschema.validators.Draft4Validator, validate_schema: bool
 ) -> None:
     if validate_schema:
         try:
             validator.validate(instance)
         except TypeError as exc:
-            raise ValidationError("Invalid schema") from exc
+            if validation.is_pattern_error(exc):
+                # Ignore errors for completely invalid schemas - it will be covered by the re-raising after this block
+                with suppress(AttributeError):
+                    status_codes = validation.find_numeric_http_status_codes(instance)
+                    if status_codes:
+                        message = _format_status_codes(status_codes)
+                        raise SchemaLoadingError(f"{NUMERIC_STATUS_CODES_MESSAGE}\n{message}") from exc
+                    # Some other pattern error
+                    raise SchemaLoadingError(NON_STRING_OBJECT_KEY) from exc
+            raise SchemaLoadingError("Invalid schema") from exc
+        except ValidationError as exc:
+            raise SchemaLoadingError("The input schema is not a valid Open API schema") from exc
 
 
 def from_pytest_fixture(

src/schemathesis/specs/openapi/validation.py

@@ -0,0 +1,17 @@
+from typing import Any, Dict, List, Tuple, Union
+
+
+def is_pattern_error(exception: TypeError) -> bool:
+    """Detect whether the input exception was caused by invalid type passed to `re.search`."""
+    # This is intentionally simplistic and do not involve any traceback analysis
+    return str(exception) == "expected string or bytes-like object"
+
+
+def find_numeric_http_status_codes(schema: Dict[str, Any]) -> List[Tuple[int, List[Union[str, int]]]]:
+    found = []
+    for path, methods in schema.get("paths", {}).items():
+        for method, definition in methods.items():
+            for key in definition.get("responses", {}):
+                if isinstance(key, int):
+                    found.append((key, [path, method]))
+    return found

test/loaders/test_openapi.py

@@ -5,8 +5,9 @@
 from flask import Response
 
 import schemathesis
+from schemathesis.exceptions import SchemaLoadingError
 from schemathesis.specs.openapi import loaders
-from schemathesis.specs.openapi.loaders import YAML_LOADING_ERROR
+from schemathesis.specs.openapi.loaders import NON_STRING_OBJECT_KEY, NUMERIC_STATUS_CODES_MESSAGE, YAML_LOADING_ERROR
 from schemathesis.specs.openapi.schemas import OpenApi30, SwaggerV20
 
 
@@ -109,3 +110,32 @@ def test_invalid_content_type(httpserver, without_content_type):
     if not without_content_type:
         # And list the actual response content type
         assert "The actual response has `text/html; charset=utf-8` Content-Type" in exc.value.args[0]
+
+
+def test_numeric_status_codes(empty_open_api_3_schema):
+    # When the API schema contains a numeric status code, which is not allowed by the spec
+    empty_open_api_3_schema["paths"] = {
+        "/foo": {
+            "get": {
+                "responses": {200: {"description": "OK"}},
+            },
+            "post": {
+                "responses": {201: {"description": "OK"}},
+            },
+        },
+    }
+    # And schema validation is enabled
+    # Then Schemathesis reports an error about numeric status codes
+    with pytest.raises(SchemaLoadingError, match=NUMERIC_STATUS_CODES_MESSAGE) as exc:
+        schemathesis.from_dict(empty_open_api_3_schema)
+    # And shows all locations of these keys
+    assert " - 200 at schema['paths']['/foo']['get']['responses']" in exc.value.args[0]
+    assert " - 201 at schema['paths']['/foo']['post']['responses']" in exc.value.args[0]
+
+
+def test_non_string_keys(empty_open_api_3_schema):
+    # If API schema contains a non-string key
+    empty_open_api_3_schema[True] = 42
+    # Then it should be reported with a proper message
+    with pytest.raises(SchemaLoadingError, match=NON_STRING_OBJECT_KEY):
+        schemathesis.from_dict(empty_open_api_3_schema)

test/test_schemas.py

@@ -2,7 +2,7 @@
 from jsonschema import ValidationError
 
 import schemathesis
-from schemathesis.exceptions import InvalidSchema
+from schemathesis.exceptions import InvalidSchema, SchemaLoadingError
 from schemathesis.specs.openapi.parameters import OpenAPI20Body
 from schemathesis.specs.openapi.schemas import InliningResolver
 from schemathesis.utils import Err, Ok
@@ -126,7 +126,7 @@ def test_schema_parsing_error(simple_schema):
     assert oks[0].method == "post"
 
 
-@pytest.mark.parametrize("validate_schema, expected_exception", ((False, InvalidSchema), (True, ValidationError)))
+@pytest.mark.parametrize("validate_schema, expected_exception", ((False, InvalidSchema), (True, SchemaLoadingError)))
 def test_not_recoverable_schema_error(simple_schema, validate_schema, expected_exception):
     # When there is an error in the API schema that leads to inability to generate any tests
     del simple_schema["paths"]
@@ -140,7 +140,7 @@ def test_schema_error_on_path(simple_schema):
     # When there is an error that affects only a subset of paths
     simple_schema["paths"] = {None: "", "/foo": {"post": RESPONSES}}
     # Then it should be rejected during loading if schema validation is enabled
-    with pytest.raises(ValidationError):
+    with pytest.raises(SchemaLoadingError):
         schemathesis.from_dict(simple_schema)
     # And should produce an `Err` instance on operation parsing
     schema = schemathesis.from_dict(simple_schema, validate_schema=False)