chore: Improve error messages for certain schema errors

Ref: #1517

docs/changelog.rst

@@ -23,7 +23,7 @@ Changelog
 - Add ``case`` as the first argument to ``AuthContext.set``. Previous calling convention is still supported. `#1788`_
 - Disable the 'explain' phase in Hypothesis to improve performance. `#1808`_
 - Do not display ``InsecureRequestWarning`` in CLI output if the user explicitly provided ``--request-tls-verify=false``. `#1780`_
-- Enhance CLI output for schema loading and internal errors, providing clearer diagnostics and guidance. `#1781`_
+- Enhance CLI output for schema loading and internal errors, providing clearer diagnostics and guidance. `#1781`_, `#1517`_
 
 Before:
 
@@ -3495,6 +3495,7 @@ Deprecated
 .. _#1538: https://github.com/schemathesis/schemathesis/issues/1538
 .. _#1526: https://github.com/schemathesis/schemathesis/issues/1526
 .. _#1518: https://github.com/schemathesis/schemathesis/issues/1518
+.. _#1517: https://github.com/schemathesis/schemathesis/issues/1517
 .. _#1514: https://github.com/schemathesis/schemathesis/issues/1514
 .. _#1485: https://github.com/schemathesis/schemathesis/issues/1485
 .. _#1464: https://github.com/schemathesis/schemathesis/issues/1464

src/schemathesis/_hypothesis.py

@@ -12,7 +12,7 @@
 
 from .auths import get_auth_storage_from_test
 from .constants import DEFAULT_DEADLINE, DataGenerationMethod
-from .exceptions import InvalidSchema
+from .exceptions import OperationSchemaError
 from .hooks import GLOBAL_HOOK_DISPATCHER, HookContext, HookDispatcher
 from .models import APIOperation, Case
 from .utils import GivenInput, combine_strategies
@@ -109,7 +109,7 @@ def add_examples(test: Callable, operation: APIOperation, hook_dispatcher: Optio
     """Add examples to the Hypothesis test, if they are specified in the schema."""
     try:
         examples: List[Case] = [get_single_example(strategy) for strategy in operation.get_strategies_from_examples()]
-    except (InvalidSchema, HypothesisRefResolutionError, Unsatisfiable):
+    except (OperationSchemaError, HypothesisRefResolutionError, Unsatisfiable):
         # Invalid schema:
         # In this case, the user didn't pass `--validate-schema=false` and see an error in the output anyway,
         # and no tests will be executed. For this reason, examples can be skipped

src/schemathesis/cli/output/default.py

@@ -138,8 +138,7 @@ def display_errors(context: ExecutionContext, event: events.Finished) -> None:
             "Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks", fg="red"
         )
     click.secho(
-        f"\nIf you need assistance in solving this, feel free to join our Discord server and report it:\n\n"
-        f"  {DISCORD_LINK}\n",
+        f"\nNeed more help?\n" f"    Join our Discord server: {DISCORD_LINK}\n",
         fg="red",
     )
 
@@ -160,7 +159,7 @@ def display_generic_errors(context: ExecutionContext, errors: List[SerializedErr
 
 def display_full_traceback_message(exception: str) -> bool:
     # Some errors should not trigger the message that suggests to show full tracebacks to the user
-    return not exception.startswith("DeadlineExceeded")
+    return not exception.startswith(("DeadlineExceeded", "OperationSchemaError"))
 
 
 def _display_error(context: ExecutionContext, error: SerializedError, seed: Optional[int] = None) -> bool:

src/schemathesis/exceptions.py

@@ -1,4 +1,5 @@
 import enum
+import json
 from dataclasses import dataclass, field
 from hashlib import sha1
 from json import JSONDecodeError
@@ -140,14 +141,41 @@ def get_timeout_error(deadline: Union[float, int]) -> Type[CheckFailed]:
 
 
 @dataclass
-class InvalidSchema(Exception):
+class OperationSchemaError(Exception):
     """Schema associated with an API operation contains an error."""
 
     __module__ = "builtins"
     message: Optional[str] = None
     path: Optional[str] = None
     method: Optional[str] = None
     full_path: Optional[str] = None
+    jsonschema_error: Optional[ValidationError] = None
+
+    @classmethod
+    def from_jsonschema_error(
+        cls, error: ValidationError, path: Optional[str], method: Optional[str], full_path: Optional[str]
+    ) -> "OperationSchemaError":
+        if error.absolute_path:
+            part = error.absolute_path[-1]
+            if isinstance(part, int) and len(error.absolute_path) > 1:
+                parent = error.absolute_path[-2]
+                message = f"Invalid definition for element at index {part} in `{parent}`"
+            else:
+                message = f"Invalid `{part}` definition"
+        else:
+            message = "Invalid schema definition"
+        error_path = " -> ".join((str(entry) for entry in error.path)) or "[root]"
+        message += f"\n\nLocation:\n    {error_path}"
+        instance = truncated_json(error.instance)
+        message += f"\n\nProblematic definition:\n{instance}"
+        message += "\n\nError details:\n    "
+        # This default message contains the instance which we already printed
+        if "is not valid under any of the given schemas" in error.message:
+            message += "The provided definition doesn't match any of the expected formats or types."
+        else:
+            message += error.message
+        message += "\n\nEnsure that the definition complies with the OpenAPI specification"
+        return cls(message, path=path, method=method, full_path=full_path, jsonschema_error=error)
 
     def as_failing_test_function(self) -> Callable:
         """Create a test function that will fail.
@@ -162,6 +190,26 @@ def actual_test(*args: Any, **kwargs: Any) -> NoReturn:
         return actual_test
 
 
+def truncated_json(data: Any, max_lines: int = 10, max_width: int = 80) -> str:
+    # Convert JSON to string with indentation
+    indent = 4
+    serialized = json.dumps(data, indent=indent)
+
+    # Split string by lines
+
+    lines = [line[: max_width - 3] + "..." if len(line) > max_width else line for line in serialized.split("\n")]
+
+    if len(lines) <= max_lines:
+        return "\n".join(lines)
+
+    truncated_lines = lines[: max_lines - 1]
+    indentation = " " * indent
+    truncated_lines.append(f"{indentation}// Output truncated...")
+    truncated_lines.append(lines[-1])
+
+    return "\n".join(truncated_lines)
+
+
 class DeadlineExceeded(Exception):
     """Test took too long to run."""
 

src/schemathesis/extra/pytest_plugin.py

@@ -14,7 +14,7 @@
 
 from .._hypothesis import create_test
 from ..constants import IS_PYTEST_ABOVE_7, IS_PYTEST_ABOVE_54, RECURSIVE_REFERENCE_ERROR_MESSAGE
-from ..exceptions import InvalidSchema, SkipTest
+from ..exceptions import OperationSchemaError, SkipTest
 from ..models import APIOperation
 from ..utils import (
     PARAMETRIZE_MARKER,
@@ -87,7 +87,9 @@ def _init_with_valid_test(_test_function: Callable, _args: Tuple, _kwargs: Dict[
     def _get_test_name(self, operation: APIOperation) -> str:
         return f"{self.name}[{operation.verbose_name}]"
 
-    def _gen_items(self, result: Result[APIOperation, InvalidSchema]) -> Generator[SchemathesisFunction, None, None]:
+    def _gen_items(
+        self, result: Result[APIOperation, OperationSchemaError]
+    ) -> Generator[SchemathesisFunction, None, None]:
         """Generate all tests for the given API operation.
 
         Could produce more than one test item if
@@ -238,7 +240,7 @@ def pytest_pyfunc_call(pyfuncitem):  # type:ignore
         try:
             outcome.get_result()
         except InvalidArgument as exc:
-            raise InvalidSchema(exc.args[0]) from None
+            raise OperationSchemaError(exc.args[0]) from None
         except HypothesisRefResolutionError:
             pytest.skip(RECURSIVE_REFERENCE_ERROR_MESSAGE)
         except SkipTest as exc:

src/schemathesis/lazy.py

@@ -14,7 +14,7 @@
 from ._compat import MultipleFailures
 from .auths import AuthStorage
 from .constants import FLAKY_FAILURE_MESSAGE, CodeSampleStyle
-from .exceptions import CheckFailed, InvalidSchema, SkipTest, get_grouped_exception
+from .exceptions import CheckFailed, OperationSchemaError, SkipTest, get_grouped_exception
 from .hooks import HookDispatcher, HookScope
 from .models import APIOperation
 from .schemas import BaseSchema
@@ -244,14 +244,15 @@ def get_exception_class() -> Type[CheckFailed]:
 SEPARATOR = "\n===================="
 
 
-def _schema_error(subtests: SubTests, error: InvalidSchema, node_id: str) -> None:
+def _schema_error(subtests: SubTests, error: OperationSchemaError, node_id: str) -> None:
     """Run a failing test, that will show the underlying problem."""
     sub_test = error.as_failing_test_function()
     # `full_path` is always available in this case
     kwargs = {"path": error.full_path}
     if error.method:
         kwargs["method"] = error.method.upper()
     subtests.item._nodeid = _get_partial_node_name(node_id, **kwargs)
+    __tracebackhide__ = True
     with subtests.test(**kwargs):
         sub_test()
 

src/schemathesis/models.py

@@ -50,7 +50,7 @@
 from .exceptions import (
     CheckFailed,
     FailureContext,
-    InvalidSchema,
+    OperationSchemaError,
     SerializationNotPossible,
     deduplicate_failed_checks,
     get_grouped_exception,
@@ -172,10 +172,10 @@ def formatted_path(self) -> str:
             # This may happen when a path template has a placeholder for variable "X", but parameter "X" is not defined
             # in the parameters list.
             # When `exc` is formatted, it is the missing key name in quotes. E.g. 'id'
-            raise InvalidSchema(f"Path parameter {exc} is not defined") from exc
+            raise OperationSchemaError(f"Path parameter {exc} is not defined") from exc
         except ValueError as exc:
             # A single unmatched `}` inside the path template may cause this
-            raise InvalidSchema(f"Malformed path template: `{self.path}`\n\n  {exc}") from exc
+            raise OperationSchemaError(f"Malformed path template: `{self.path}`\n\n  {exc}") from exc
 
     def get_full_base_url(self) -> Optional[str]:
         """Create a full base url, adding "localhost" for WSGI apps."""
@@ -1108,7 +1108,7 @@ class TestResultSet:
     __test__ = False
 
     results: List[TestResult] = field(default_factory=list)
-    generic_errors: List[InvalidSchema] = field(default_factory=list)
+    generic_errors: List[OperationSchemaError] = field(default_factory=list)
     warnings: List[str] = field(default_factory=list)
 
     def __iter__(self) -> Iterator[TestResult]:

src/schemathesis/runner/impl/core.py

@@ -14,6 +14,7 @@
 from _pytest.logging import LogCaptureHandler, catching_logs
 from hypothesis.errors import HypothesisException, InvalidArgument
 from hypothesis_jsonschema._canonicalise import HypothesisRefResolutionError
+from jsonschema.exceptions import ValidationError
 from requests.auth import HTTPDigestAuth, _basic_auth_str
 
 from ... import failures, hooks
@@ -29,8 +30,8 @@
     CheckFailed,
     DeadlineExceeded,
     InvalidRegularExpression,
-    InvalidSchema,
     NonCheckError,
+    OperationSchemaError,
     SkipTest,
     get_grouped_exception,
 )
@@ -184,7 +185,7 @@ def _run_tests(
                         headers=headers,
                         **kwargs,
                     )
-                except InvalidSchema as exc:
+                except OperationSchemaError as exc:
                     yield from handle_schema_error(
                         exc,
                         results,
@@ -229,7 +230,7 @@ def finish(self) -> events.ExecutionEvent:
 
 
 def handle_schema_error(
-    error: InvalidSchema,
+    error: OperationSchemaError,
     results: TestResultSet,
     data_generation_methods: Iterable[DataGenerationMethod],
     recursion_level: int,
@@ -364,8 +365,8 @@ def run_test(
     except SkipTest:
         status = Status.skip
         result.mark_skipped()
-    except AssertionError as exc:  # comes from `hypothesis-jsonschema`
-        error = reraise(exc)
+    except AssertionError:  # comes from `hypothesis-jsonschema`
+        error = reraise(operation)
         status = Status.error
         result.add_error(error)
     except HypothesisRefResolutionError:
@@ -466,16 +467,14 @@ def get_invalid_regular_expression_message(warnings: List[WarningMessage]) -> Op
     return None
 
 
-def reraise(error: AssertionError) -> InvalidSchema:
-    traceback = format_exception(error, True)
-    if "assert type_ in TYPE_STRINGS" in traceback:
-        message = "Invalid type name"
-    else:
-        message = "Unknown schema error"
+def reraise(operation: APIOperation) -> OperationSchemaError:
     try:
-        raise InvalidSchema(message) from error
-    except InvalidSchema as exc:
-        return exc
+        operation.schema.validate()
+    except ValidationError as exc:
+        return OperationSchemaError.from_jsonschema_error(
+            exc, path=operation.path, method=operation.method, full_path=operation.schema.get_full_path(operation.path)
+        )
+    return OperationSchemaError("Unknown schema error")
 
 
 def deduplicate_errors(errors: List[Exception]) -> Generator[Exception, None, None]:

src/schemathesis/schemas.py

@@ -38,7 +38,7 @@
 from ._hypothesis import create_test
 from .auths import AuthStorage
 from .constants import DEFAULT_DATA_GENERATION_METHODS, CodeSampleStyle, DataGenerationMethod
-from .exceptions import InvalidSchema, UsageError
+from .exceptions import OperationSchemaError, UsageError
 from .hooks import HookContext, HookDispatcher, HookScope, dispatch
 from .models import APIOperation, Case
 from .stateful import APIStateMachine, Stateful, StatefulTest
@@ -155,6 +155,9 @@ def get_base_url(self) -> str:
             return base_url.rstrip("/")
         return self._build_base_url()
 
+    def validate(self) -> None:
+        raise NotImplementedError
+
     @property
     def operations(self) -> Dict[str, MethodsDict]:
         if not hasattr(self, "_operations"):
@@ -168,7 +171,7 @@ def operations_count(self) -> int:
 
     def get_all_operations(
         self, hooks: Optional[HookDispatcher] = None
-    ) -> Generator[Result[APIOperation, InvalidSchema], None, None]:
+    ) -> Generator[Result[APIOperation, OperationSchemaError], None, None]:
         raise NotImplementedError
 
     def get_strategies_from_examples(self, operation: APIOperation) -> List[SearchStrategy[Case]]:
@@ -197,7 +200,7 @@ def get_all_tests(
         as_strategy_kwargs: Optional[Dict[str, Any]] = None,
         hooks: Optional[HookDispatcher] = None,
         _given_kwargs: Optional[Dict[str, GivenInput]] = None,
-    ) -> Generator[Result[Tuple[APIOperation, Callable], InvalidSchema], None, None]:
+    ) -> Generator[Result[Tuple[APIOperation, Callable], OperationSchemaError], None, None]:
         """Generate all operations and Hypothesis tests for them."""
         for result in self.get_all_operations(hooks=hooks):
             if isinstance(result, Ok):
@@ -406,7 +409,7 @@ def _get_payload_schema(self, definition: Dict[str, Any], media_type: str) -> Op
 
 
 def operations_to_dict(
-    operations: Generator[Result[APIOperation, InvalidSchema], None, None]
+    operations: Generator[Result[APIOperation, OperationSchemaError], None, None]
 ) -> Dict[str, MethodsDict]:
     output: Dict[str, MethodsDict] = {}
     for result in operations:

src/schemathesis/specs/graphql/schemas.py

@@ -15,7 +15,7 @@
 from ...auths import AuthStorage
 from ...checks import not_a_server_error
 from ...constants import DataGenerationMethod
-from ...exceptions import InvalidSchema
+from ...exceptions import OperationSchemaError
 from ...hooks import (
     GLOBAL_HOOK_DISPATCHER,
     HookContext,
@@ -146,7 +146,7 @@ def operations_count(self) -> int:
 
     def get_all_operations(
         self, hooks: Optional[HookDispatcher] = None
-    ) -> Generator[Result[APIOperation, InvalidSchema], None, None]:
+    ) -> Generator[Result[APIOperation, OperationSchemaError], None, None]:
         schema = self.client_schema
         for root_type, operation_type in (
             (RootType.QUERY, schema.query_type),

src/schemathesis/specs/openapi/_hypothesis.py

@@ -14,7 +14,7 @@
 
 from ... import auths, serializers, utils
 from ...constants import DataGenerationMethod
-from ...exceptions import InvalidSchema, SerializationNotPossible
+from ...exceptions import OperationSchemaError, SerializationNotPossible
 from ...hooks import HookContext, HookDispatcher, apply_to_all_dispatchers
 from ...models import APIOperation, Case, cant_serialize
 from ...types import NotSet
@@ -186,7 +186,7 @@ def get_case_strategy(
         body_ = ValueContainer(value=body, location="body", generator=None)
 
     if operation.schema.validate_schema and operation.method.upper() == "GET" and operation.body:
-        raise InvalidSchema("Body parameters are defined for GET request.")
+        raise OperationSchemaError("Body parameters are defined for GET request.")
     # If we need to generate negative cases but no generated values were negated, then skip the whole test
     if generator.is_negative and not any_negated_values([query_, cookies_, headers_, path_parameters_, body_]):
         skip(operation.verbose_name)

src/schemathesis/specs/openapi/parameters.py

@@ -2,7 +2,7 @@
 from dataclasses import dataclass
 from typing import Any, ClassVar, Dict, Iterable, List, Optional, Tuple
 
-from ...exceptions import InvalidSchema
+from ...exceptions import OperationSchemaError
 from ...models import APIOperation
 from ...parameters import Parameter
 from .converter import to_json_schema_recursive
@@ -412,7 +412,7 @@ def get_parameter_schema(operation: APIOperation, data: Dict[str, Any]) -> Dict[
     try:
         content = data["content"]
     except KeyError as exc:
-        raise InvalidSchema(
+        raise OperationSchemaError(
             MISSING_SCHEMA_OR_CONTENT_MESSAGE.format(location=data.get("in", ""), name=data.get("name", "<UNKNOWN>")),
             path=operation.path,
             method=operation.method,