From 8c4fe09c94e65cec9814d8e36fdf379e2810db43 Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Thu, 2 Nov 2023 14:43:05 +0100 Subject: [PATCH] chore: Clarify CLI error messages for issues coming from network and configuration Ref: #1607 #1835 --- docs/changelog.rst | 3 + src/schemathesis/cli/output/default.py | 107 +++++++++--- src/schemathesis/exceptions.py | 64 ++++++- src/schemathesis/loaders.py | 17 +- src/schemathesis/runner/events.py | 19 ++- src/schemathesis/runner/impl/core.py | 10 +- src/schemathesis/runner/serialization.py | 78 ++++++++- src/schemathesis/specs/openapi/_hypothesis.py | 4 +- .../test_in_cli[1].raw | 44 +++++ .../test_in_cli[2].raw | 43 +++++ .../test_in_cli[Open API 3.0].raw | 29 ++++ .../test_connection_error[Open API 2.0-1].raw | 36 ++++ .../test_connection_error[Open API 2.0-2].raw | 35 ++++ .../test_connection_error[Open API 3.0-1].raw | 36 ++++ .../test_connection_error[Open API 3.0-2].raw | 35 ++++ ...st_get_request_with_body[Open API 2.0].raw | 31 ++++ ...st_get_request_with_body[Open API 3.0].raw | 31 ++++ ...esis_failed_event[real-Open API 2.0-1].raw | 31 ++++ ...esis_failed_event[real-Open API 2.0-2].raw | 30 ++++ ...esis_failed_event[real-Open API 3.0-1].raw | 31 ++++ ...esis_failed_event[real-Open API 3.0-2].raw | 30 ++++ ...esis_failed_event[wsgi-Open API 2.0-1].raw | 31 ++++ ...esis_failed_event[wsgi-Open API 2.0-2].raw | 30 ++++ ...esis_failed_event[wsgi-Open API 3.0-1].raw | 31 ++++ ...esis_failed_event[wsgi-Open API 3.0-2].raw | 30 ++++ ...any of the expected formats or types.].raw | 47 +++++ ...n[3.1.0-'type' is a required property].raw | 47 +++++ ...issing_content_and_schema[cookie-None].raw | 27 +++ ..._and_schema[cookie-http127.0.0.1apiv2].raw | 27 +++ ...issing_content_and_schema[header-None].raw | 27 +++ ..._and_schema[header-http127.0.0.1apiv2].raw | 27 +++ ..._missing_content_and_schema[path-None].raw | 27 +++ ...nt_and_schema[path-http127.0.0.1apiv2].raw | 27 +++ ...missing_content_and_schema[query-None].raw | 27 +++ ...t_and_schema[query-http127.0.0.1apiv2].raw | 27 +++ .../test_schema_not_available[1].raw | 10 ++ .../test_schema_not_available[2].raw | 10 ++ ...ble_reference_with_disabled_validation.raw | 44 +++++ ...est_unsatisfiable[real-Open API 2.0-1].raw | 34 ++++ ...est_unsatisfiable[real-Open API 2.0-2].raw | 34 ++++ ...est_unsatisfiable[real-Open API 3.0-1].raw | 34 ++++ ...est_unsatisfiable[real-Open API 3.0-2].raw | 34 ++++ ...est_unsatisfiable[wsgi-Open API 2.0-1].raw | 34 ++++ ...est_unsatisfiable[wsgi-Open API 2.0-2].raw | 34 ++++ ...est_unsatisfiable[wsgi-Open API 3.0-1].raw | 34 ++++ ...est_unsatisfiable[wsgi-Open API 3.0-2].raw | 34 ++++ .../test_commands/test_unsupported_regex.raw | 32 ++++ .../test_wait_for_schema_not_enough.raw | 2 +- test/cli/output/test_default.py | 71 -------- test/cli/test_commands.py | 160 +++--------------- test/cli/test_junitxml.py | 2 +- test/conftest.py | 27 ++- test/runner/test_runner.py | 2 +- test/test_hypothesis.py | 2 +- test/test_lazy.py | 2 +- test/test_parameters.py | 2 +- test/test_recoverable_errors.py | 15 +- test/test_serialization.py | 6 + 58 files changed, 1547 insertions(+), 288 deletions(-) create mode 100644 test/__snapshots__/test_recoverable_errors/test_in_cli[1].raw create mode 100644 test/__snapshots__/test_recoverable_errors/test_in_cli[2].raw create mode 100644 test/__snapshots__/test_serialization/test_in_cli[Open API 3.0].raw create mode 100644 test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 2.0].raw create mode 100644 test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 3.0].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.0.2-The provided definition doesn't match any of the expected formats or types.].raw create mode 100644 test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.1.0-'type' is a required property].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-None].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-http127.0.0.1apiv2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-None].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-http127.0.0.1apiv2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-None].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-http127.0.0.1apiv2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-None].raw create mode 100644 test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-http127.0.0.1apiv2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_schema_not_available[1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_schema_not_available[2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unresolvable_reference_with_disabled_validation.raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-1].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-2].raw create mode 100644 test/cli/__snapshots__/test_commands/test_unsupported_regex.raw diff --git a/docs/changelog.rst b/docs/changelog.rst index 16d423fc11..c77fb2bdae 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Changelog - Clarify CLI error messages for loading hooks and WSGI applications. - Clarify CLI option docstrings. - Provide an error message if an internal error happened inside CLI event handler. +- Clarify CLI error messages for issues coming from network and configuration. `#1607`_, `#1835`_ .. _v3.20.2: @@ -3508,6 +3509,7 @@ Deprecated .. _0.3.0: https://github.com/schemathesis/schemathesis/compare/v0.2.0...v0.3.0 .. _0.2.0: https://github.com/schemathesis/schemathesis/compare/v0.1.0...v0.2.0 +.. _#1835: https://github.com/schemathesis/schemathesis/issues/1835 .. _#1820: https://github.com/schemathesis/schemathesis/issues/1820 .. _#1819: https://github.com/schemathesis/schemathesis/issues/1819 .. _#1808: https://github.com/schemathesis/schemathesis/issues/1808 @@ -3542,6 +3544,7 @@ Deprecated .. _#1627: https://github.com/schemathesis/schemathesis/issues/1627 .. _#1625: https://github.com/schemathesis/schemathesis/issues/1625 .. _#1614: https://github.com/schemathesis/schemathesis/issues/1614 +.. _#1607: https://github.com/schemathesis/schemathesis/issues/1607 .. _#1602: https://github.com/schemathesis/schemathesis/issues/1602 .. _#1592: https://github.com/schemathesis/schemathesis/issues/1592 .. _#1591: https://github.com/schemathesis/schemathesis/issues/1591 diff --git a/src/schemathesis/cli/output/default.py b/src/schemathesis/cli/output/default.py index 83357e1c35..b6c1b42fbd 100644 --- a/src/schemathesis/cli/output/default.py +++ b/src/schemathesis/cli/output/default.py @@ -20,6 +20,7 @@ FALSE_VALUES, ISSUE_TRACKER_URL, ) +from ...exceptions import RuntimeErrorType from ...experimental import GLOBAL_EXPERIMENTS from ...models import Response, Status from ...runner import events @@ -130,7 +131,13 @@ def display_errors(context: ExecutionContext, event: events.Finished) -> None: display_section_name("ERRORS") should_display_full_traceback_message = False - for result in context.results: + if context.workers_num > 1: + # Events may come out of order when multiple workers are involved + # Sort them to get a stable output + results = sorted(context.results, key=lambda r: r.verbose_name) + else: + results = context.results + for result in results: if not result.has_errors: continue should_display_full_traceback_message |= display_single_error(context, result) @@ -138,7 +145,8 @@ def display_errors(context: ExecutionContext, event: events.Finished) -> None: display_generic_errors(context, event.generic_errors) if should_display_full_traceback_message and not context.show_errors_tracebacks: click.secho( - "Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks", fg="red" + "\nAdd this option to your command line parameters to see full tracebacks: --show-errors-tracebacks", + fg="red", ) click.secho( f"\nNeed more help?\n" f" Join our Discord server: {DISCORD_LINK}", @@ -160,23 +168,58 @@ def display_generic_errors(context: ExecutionContext, errors: List[SerializedErr _display_error(context, error) -def display_full_traceback_message(exception: str) -> bool: +def display_full_traceback_message(error: SerializedError) -> bool: # Some errors should not trigger the message that suggests to show full tracebacks to the user - return not exception.startswith(("DeadlineExceeded", "OperationSchemaError")) + return not error.exception.startswith( + ("DeadlineExceeded", "OperationSchemaError", "requests.exceptions", "SerializationNotPossible") + ) + + +def bold(option: str) -> str: + return click.style(option, bold=True) + + +DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}." +DISABLE_SCHEMA_VALIDATION_SUGGESTION = ( + f"Bypass validation using {bold('`--validate-schema=false`')}. " f"Caution: May cause unexpected errors." +) + +RUNTIME_ERROR_SUGGESTIONS = { + RuntimeErrorType.CONNECTION_SSL: DISABLE_SSL_SUGGESTION, + RuntimeErrorType.HYPOTHESIS_DEADLINE_EXCEEDED: ( + f"Adjust the deadline using {bold('`--hypothesis-deadline=MILLIS`')} or " + f"disable with {bold('`--hypothesis-deadline=None`')}." + ), + RuntimeErrorType.HYPOTHESIS_UNSATISFIABLE: "Examine the schema for inconsistencies and consider simplifying it.", + RuntimeErrorType.SCHEMA_BODY_IN_GET_REQUEST: DISABLE_SCHEMA_VALIDATION_SUGGESTION, + RuntimeErrorType.SCHEMA_INVALID_REGULAR_EXPRESSION: "Ensure your regex is compatible with Python's syntax. " + "For guidance, visit: https://docs.python.org/3/library/re.html", +} def _display_error(context: ExecutionContext, error: SerializedError) -> bool: - if context.show_errors_tracebacks: - message = error.exception_with_traceback + if error.title: + if error.type == RuntimeErrorType.SCHEMA_GENERIC: + click.secho("Schema Error", fg="red", bold=True) + else: + click.secho(error.title, fg="red", bold=True) + click.echo() + if error.message: + click.echo(error.message) + elif error.message: + click.echo(error.message) else: - message = error.exception - if error.exception.startswith("DeadlineExceeded"): - message += ( - "Consider extending the deadline with the `--hypothesis-deadline` CLI option.\n" - "You can disable it completely with `--hypothesis-deadline=None`.\n" - ) - click.secho(message, fg="red") - return display_full_traceback_message(error.exception) + click.echo(error.exception) + if error.extras: + extras = error.extras + elif context.show_errors_tracebacks: + extras = _split_traceback(error.exception_with_traceback) + else: + extras = [] + _display_extras(extras) + suggestion = RUNTIME_ERROR_SUGGESTIONS.get(error.type) + _maybe_display_tip(suggestion) + return display_full_traceback_message(error) def display_failures(context: ExecutionContext, event: events.Finished) -> None: @@ -480,13 +523,9 @@ def display_check_result(check_name: str, results: Dict[Union[str, Status], int] VERIFY_URL_SUGGESTION = "Verify that the URL points directly to the Open API schema" -def bold(option: str) -> str: - return click.style(option, bold=True) - - SCHEMA_ERROR_SUGGESTIONS = { # SSL-specific connection issue - SchemaErrorType.CONNECTION_SSL: f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}.", + SchemaErrorType.CONNECTION_SSL: DISABLE_SSL_SUGGESTION, # Other connection problems SchemaErrorType.CONNECTION_OTHER: f"Use {bold('`--wait-for-schema=NUM`')} to wait up to NUM seconds for schema availability.", # Response issues @@ -496,7 +535,7 @@ def bold(option: str) -> str: # OpenAPI specification issues SchemaErrorType.OPEN_API_UNSPECIFIED_VERSION: f"Include the version in the schema or manually set it with {bold('`--force-schema-version`')}.", SchemaErrorType.OPEN_API_UNSUPPORTED_VERSION: f"Proceed with {bold('`--force-schema-version`')}. Caution: May not be fully supported.", - SchemaErrorType.OPEN_API_INVALID_SCHEMA: f"Bypass validation using {bold('`--validate-schema=false`')}. Caution: May cause unexpected errors.", + SchemaErrorType.OPEN_API_INVALID_SCHEMA: DISABLE_SCHEMA_VALIDATION_SUGGESTION, # YAML specific issues SchemaErrorType.YAML_NUMERIC_STATUS_CODES: "Convert numeric status codes to strings.", SchemaErrorType.YAML_NON_STRING_KEYS: "Convert non-string keys to strings.", @@ -509,6 +548,23 @@ def should_skip_suggestion(context: ExecutionContext, event: events.InternalErro return event.subtype == SchemaErrorType.CONNECTION_OTHER and context.wait_for_schema is not None +def _split_traceback(traceback: str) -> List[str]: + return [entry for entry in traceback.splitlines() if entry] + + +def _display_extras(extras: List[str]) -> None: + if extras: + click.echo() + for extra in extras: + click.secho(f" {extra}") + + +def _maybe_display_tip(suggestion: Optional[str]) -> None: + # Display suggestion if any + if suggestion is not None: + click.secho(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}") + + def display_internal_error(context: ExecutionContext, event: events.InternalError) -> None: click.secho(event.title, fg="red", bold=True) click.echo() @@ -516,13 +572,10 @@ def display_internal_error(context: ExecutionContext, event: events.InternalErro if event.type == InternalErrorType.SCHEMA: extras = event.extras elif context.show_errors_tracebacks: - extras = [entry for entry in event.exception_with_traceback.splitlines() if entry] + extras = _split_traceback(event.exception_with_traceback) else: extras = [event.exception] - if extras: - click.echo() - for extra in extras: - click.secho(f" {extra}") + _display_extras(extras) if not should_skip_suggestion(context, event): if event.type == InternalErrorType.SCHEMA and isinstance(event.subtype, SchemaErrorType): suggestion = SCHEMA_ERROR_SUGGESTIONS.get(event.subtype) @@ -532,9 +585,7 @@ def display_internal_error(context: ExecutionContext, event: events.InternalErro ) else: suggestion = f"To see full tracebacks, add {bold('`--show-errors-tracebacks`')} to your CLI options" - # Display suggestion if any - if suggestion is not None: - click.secho(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}") + _maybe_display_tip(suggestion) def handle_initialized(context: ExecutionContext, event: events.Initialized) -> None: diff --git a/src/schemathesis/exceptions.py b/src/schemathesis/exceptions.py index f239300cc4..4eb5acc44b 100644 --- a/src/schemathesis/exceptions.py +++ b/src/schemathesis/exceptions.py @@ -1,6 +1,7 @@ from __future__ import annotations import enum import json +import re import traceback from dataclasses import dataclass, field from hashlib import sha1 @@ -15,6 +16,7 @@ import hypothesis.errors from jsonschema import RefResolutionError, ValidationError from .transports.responses import GenericResponse + from requests import RequestException class CheckFailed(AssertionError): @@ -206,6 +208,22 @@ def actual_test(*args: Any, **kwargs: Any) -> NoReturn: return actual_test +@dataclass +class BodyInGetRequestError(OperationSchemaError): + __module__ = "builtins" + + +class InvalidRegularExpression(OperationSchemaError): + __module__ = "builtins" + + @classmethod + def from_hypothesis_jsonschema_message(cls, message: str) -> "InvalidRegularExpression": + match = re.search(r"pattern='(.*?)'.*?\((.*?)\)", message) + if match: + message = f"Invalid regular expression. Pattern `{match.group(1)}` is not recognized - `{match.group(2)}`" + return cls(message) + + def truncated_json(data: Any, max_lines: int = 10, max_width: int = 80) -> str: # Convert JSON to string with indentation indent = 4 @@ -241,7 +259,26 @@ def from_exc(cls, exc: hypothesis.errors.DeadlineExceeded) -> "DeadlineExceeded" @enum.unique -class SchemaErrorType(enum.Enum): +class RuntimeErrorType(str, enum.Enum): + # Connection related issues + CONNECTION_SSL = "connection_ssl" + CONNECTION_OTHER = "connection_other" + NETWORK_OTHER = "network_other" + + # Hypothesis issues + HYPOTHESIS_DEADLINE_EXCEEDED = "hypothesis_deadline_exceeded" + HYPOTHESIS_UNSATISFIABLE = "hypothesis_unsatisfiable" + + SCHEMA_BODY_IN_GET_REQUEST = "schema_body_in_get_request" + SCHEMA_INVALID_REGULAR_EXPRESSION = "schema_invalid_regular_expression" + SCHEMA_GENERIC = "schema_generic" + + # Unclassified + UNCLASSIFIED = "unclassified" + + +@enum.unique +class SchemaErrorType(str, enum.Enum): # Connection related issues CONNECTION_SSL = "connection_ssl" CONNECTION_OTHER = "connection_other" @@ -352,10 +389,6 @@ def for_media_type(cls, media_type: str) -> "SerializationNotPossible": return cls(SERIALIZATION_FOR_TYPE_IS_NOT_POSSIBLE_MESSAGE.format(media_type)) -class InvalidRegularExpression(Exception): - __module__ = "builtins" - - class UsageError(Exception): """Incorrect usage of Schemathesis functions.""" @@ -384,3 +417,24 @@ def extract_nth_traceback(trace: Optional[TracebackType], n: int) -> Optional[Tr trace = trace.tb_next depth += 1 return trace + + +def remove_ssl_line_number(text: str) -> str: + return re.sub(r"\(_ssl\.c:\d+\)", "", text) + + +def extract_requests_exception_details(exc: RequestException) -> Tuple[str, List[str]]: + from requests.exceptions import SSLError, ConnectionError + + if isinstance(exc, SSLError): + message = "SSL verification problem" + reason = str(exc.args[0].reason) + extra = [remove_ssl_line_number(reason)] + elif isinstance(exc, ConnectionError): + message = "Connection failed" + _, reason = exc.args[0].reason.args[0].split(":", maxsplit=1) + extra = [reason.strip()] + else: + message = "Network problem" + extra = [] + return message, extra diff --git a/src/schemathesis/loaders.py b/src/schemathesis/loaders.py index 691b738b23..79ae1316fc 100644 --- a/src/schemathesis/loaders.py +++ b/src/schemathesis/loaders.py @@ -4,7 +4,7 @@ from functools import lru_cache from typing import Callable, TypeVar, cast, TYPE_CHECKING, TextIO, Any, Dict, Type, BinaryIO -from .exceptions import SchemaError, SchemaErrorType +from .exceptions import SchemaError, SchemaErrorType, extract_requests_exception_details if TYPE_CHECKING: from .transports.responses import GenericResponse @@ -13,10 +13,6 @@ R = TypeVar("R", bound="GenericResponse") -def remove_ssl_line_number(text: str) -> str: - return re.sub(r"\(_ssl\.c:\d+\)", "", text) - - def load_schema_from_url(loader: Callable[[], R]) -> R: import requests @@ -25,20 +21,13 @@ def load_schema_from_url(loader: Callable[[], R]) -> R: except requests.RequestException as exc: request = cast(requests.PreparedRequest, exc.request) if isinstance(exc, requests.exceptions.SSLError): - message = "SSL verification problem" type_ = SchemaErrorType.CONNECTION_SSL - reason = str(exc.args[0].reason) - extra = [remove_ssl_line_number(reason)] elif isinstance(exc, requests.exceptions.ConnectionError): - message = "Connection failed" type_ = SchemaErrorType.CONNECTION_OTHER - _, reason = exc.args[0].reason.args[0].split(":", maxsplit=1) - extra = [reason.strip()] else: - message = "Network problem" type_ = SchemaErrorType.NETWORK_OTHER - extra = [] - raise SchemaError(message=message, type=type_, url=request.url, response=exc.response, extras=extra) from exc + message, extras = extract_requests_exception_details(exc) + raise SchemaError(message=message, type=type_, url=request.url, response=exc.response, extras=extras) from exc _raise_for_status(response) return response diff --git a/src/schemathesis/runner/events.py b/src/schemathesis/runner/events.py index 5c7362b52b..a57579939b 100644 --- a/src/schemathesis/runner/events.py +++ b/src/schemathesis/runner/events.py @@ -7,7 +7,7 @@ from ..internal.datetime import current_datetime from ..generation import DataGenerationMethod -from ..exceptions import SchemaError, SchemaErrorType, format_exception +from ..exceptions import SchemaError, SchemaErrorType, format_exception, RuntimeErrorType from .serialization import SerializedError, SerializedTestResult @@ -205,7 +205,7 @@ def from_schema_error(cls, error: SchemaError) -> "InternalError": subtype=error.type, title="Schema Loading Error", message=error.message, - extra=error.extras, + extras=error.extras, ) @classmethod @@ -216,7 +216,7 @@ def from_exc(cls, exc: Exception) -> "InternalError": subtype=None, title="Test Execution Error", message="An internal error occurred during the test run", - extra=[], + extras=[], ) @classmethod @@ -227,7 +227,7 @@ def with_exception( subtype: Optional[SchemaErrorType], title: str, message: str, - extra: List[str], + extras: List[str], ) -> "InternalError": exception_type = f"{exc.__class__.__module__}.{exc.__class__.__qualname__}" exception = format_exception(exc) @@ -237,7 +237,7 @@ def with_exception( subtype=subtype, title=title, message=message, - extras=extra, + extras=extras, exception_type=exception_type, exception=exception, exception_with_traceback=exception_with_traceback, @@ -284,7 +284,14 @@ def from_results(cls, results: TestResultSet, running_time: float) -> "Finished" is_empty=results.is_empty, total=results.total, generic_errors=[ - SerializedError.from_error(exception=error, title=error.full_path) for error in results.generic_errors + SerializedError.with_exception( + type_=RuntimeErrorType.SCHEMA_GENERIC, + exception=error, + title=error.full_path, + message=error.message, + extras=[], + ) + for error in results.generic_errors ], warnings=results.warnings, running_time=running_time, diff --git a/src/schemathesis/runner/impl/core.py b/src/schemathesis/runner/impl/core.py index 8cfb0ac869..687eb9e3ad 100644 --- a/src/schemathesis/runner/impl/core.py +++ b/src/schemathesis/runner/impl/core.py @@ -1,5 +1,6 @@ from __future__ import annotations import logging +import re import threading import time import unittest @@ -353,7 +354,7 @@ def run_test( except hypothesis.errors.Unsatisfiable: # We need more clear error message here status = Status.error - result.add_error(hypothesis.errors.Unsatisfiable("Unable to satisfy schema parameters for this API operation")) + result.add_error(hypothesis.errors.Unsatisfiable("Failed to generate test cases for this API operation")) except KeyboardInterrupt: yield events.Interrupted() return @@ -372,7 +373,7 @@ def run_test( message = get_invalid_regular_expression_message(warnings) if message: # `hypothesis-jsonschema` emits a warning on invalid regular expression syntax - result.add_error(InvalidRegularExpression(message)) + result.add_error(InvalidRegularExpression.from_hypothesis_jsonschema_message(message)) else: result.add_error(error) except hypothesis.errors.DeadlineExceeded as error: @@ -472,11 +473,16 @@ def reraise(operation: APIOperation) -> OperationSchemaError: return OperationSchemaError("Unknown schema error") +MEMORY_ADDRESS_RE = re.compile("0x[0-9a-fA-F]+") + + def deduplicate_errors(errors: List[Exception]) -> Generator[Exception, None, None]: """Deduplicate errors by their messages + tracebacks.""" seen = set() for error in errors: message = format_exception(error, True) + # Replace memory addresses with a fixed string + message = MEMORY_ADDRESS_RE.sub("0xbaaaaaaaaaad", message) if message in seen: continue seen.add(message) diff --git a/src/schemathesis/runner/serialization.py b/src/schemathesis/runner/serialization.py index 1fbfe183dd..6d2eac78a0 100644 --- a/src/schemathesis/runner/serialization.py +++ b/src/schemathesis/runner/serialization.py @@ -9,7 +9,18 @@ from ..transports import serialize_payload from ..code_samples import get_excluded_headers -from ..exceptions import FailureContext, InternalError, make_unique_by_key, format_exception +from ..exceptions import ( + FailureContext, + InternalError, + make_unique_by_key, + format_exception, + extract_requests_exception_details, + RuntimeErrorType, + DeadlineExceeded, + OperationSchemaError, + BodyInGetRequestError, + InvalidRegularExpression, +) from ..models import Case, Check, Interaction, Request, Response, Status, TestResult if TYPE_CHECKING: @@ -155,22 +166,77 @@ def get_serialized_history(case: Case) -> List[SerializedHistoryEntry]: @dataclass class SerializedError: + type: RuntimeErrorType + title: Optional[str] + message: Optional[str] + extras: List[str] + + # Exception info exception: str exception_with_traceback: str - title: Optional[str] @classmethod - def from_error( + def with_exception( cls, + type_: RuntimeErrorType, + title: Optional[str], + message: Optional[str], + extras: List[str], exception: Exception, - title: Optional[str] = None, ) -> "SerializedError": return cls( + type=type_, + title=title, + message=message, + extras=extras, exception=format_exception(exception), exception_with_traceback=format_exception(exception, True), - title=title, ) + @classmethod + def from_exception(cls, exception: Exception) -> "SerializedError": + import requests + import hypothesis.errors + + title = "Runtime Error" + message: Optional[str] + if isinstance(exception, requests.RequestException): + if isinstance(exception, requests.exceptions.SSLError): + type_ = RuntimeErrorType.CONNECTION_SSL + elif isinstance(exception, requests.exceptions.ConnectionError): + type_ = RuntimeErrorType.CONNECTION_OTHER + else: + type_ = RuntimeErrorType.NETWORK_OTHER + message, extras = extract_requests_exception_details(exception) + title = "Network Error" + elif isinstance(exception, DeadlineExceeded): + type_ = RuntimeErrorType.HYPOTHESIS_DEADLINE_EXCEEDED + message = str(exception).strip() + extras = [] + elif isinstance(exception, hypothesis.errors.Unsatisfiable): + type_ = RuntimeErrorType.HYPOTHESIS_UNSATISFIABLE + message = f"{exception}. Possible reasons:" + extras = [ + "- Contradictory schema constraints, such as a minimum value exceeding the maximum.", + "- Excessive schema complexity, which hinders parameter generation.", + ] + title = "Schema Error" + elif isinstance(exception, OperationSchemaError): + if isinstance(exception, BodyInGetRequestError): + type_ = RuntimeErrorType.SCHEMA_BODY_IN_GET_REQUEST + elif isinstance(exception, InvalidRegularExpression): + type_ = RuntimeErrorType.SCHEMA_INVALID_REGULAR_EXPRESSION + else: + type_ = RuntimeErrorType.SCHEMA_GENERIC + message = exception.message + extras = [] + title = "Schema Error" + else: + type_ = RuntimeErrorType.UNCLASSIFIED + message = str(exception) + extras = [] + return cls.with_exception(type_=type_, exception=exception, title=title, message=message, extras=extras) + @dataclass class SerializedInteraction: @@ -226,7 +292,7 @@ def from_test_result(cls, result: TestResult) -> "SerializedTestResult": data_generation_method=[m.as_short_name() for m in result.data_generation_method], checks=[SerializedCheck.from_check(check) for check in result.checks], logs=[formatter.format(record) for record in result.logs], - errors=[SerializedError.from_error(error) for error in result.errors], + errors=[SerializedError.from_exception(error) for error in result.errors], interactions=[SerializedInteraction.from_interaction(interaction) for interaction in result.interactions], ) diff --git a/src/schemathesis/specs/openapi/_hypothesis.py b/src/schemathesis/specs/openapi/_hypothesis.py index 069b645ac0..591b1ff14b 100644 --- a/src/schemathesis/specs/openapi/_hypothesis.py +++ b/src/schemathesis/specs/openapi/_hypothesis.py @@ -18,7 +18,7 @@ from ... import auths, serializers from ...generation import DataGenerationMethod from ...internal.copy import fast_deepcopy -from ...exceptions import OperationSchemaError, SerializationNotPossible +from ...exceptions import SerializationNotPossible, BodyInGetRequestError from ...hooks import HookContext, HookDispatcher, apply_to_all_dispatchers from ...models import APIOperation, Case, cant_serialize from ...transports.headers import has_invalid_characters, is_latin_1_encodable @@ -169,7 +169,7 @@ def get_case_strategy( body_ = ValueContainer(value=body, location="body", generator=None) if operation.schema.validate_schema and operation.method.upper() == "GET" and operation.body: - raise OperationSchemaError("Body parameters are defined for GET request.") + raise BodyInGetRequestError("GET requests should not contain body parameters.") # If we need to generate negative cases but no generated values were negated, then skip the whole test if generator.is_negative and not any_negated_values([query_, cookies_, headers_, path_parameters_, body_]): skip(operation.verbose_name) diff --git a/test/__snapshots__/test_recoverable_errors/test_in_cli[1].raw b/test/__snapshots__/test_recoverable_errors/test_in_cli[1].raw new file mode 100644 index 0000000000..ea70425338 --- /dev/null +++ b/test/__snapshots__/test_recoverable_errors/test_in_cli[1].raw @@ -0,0 +1,44 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.yaml +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 2 + +GET /bar . [ 50%] +POST /bar E [100%] + +==================================== ERRORS ==================================== +__________________________________ POST /bar ___________________________________ +Schema Error + +Unresolvable JSON pointer in the schema + +Error details: + JSON pointer: 'components/UnknownParameter' + This typically means that the schema is referencing a component that doesn't exist. + +Ensure that the definition complies with the OpenAPI specification +_____________________________________ /foo _____________________________________ +Schema Error + +Unresolvable JSON pointer in the schema + +Error details: + JSON pointer: 'components/UnknownMethods' + This typically means that the schema is referencing a component that doesn't exist. + +Ensure that the definition complies with the OpenAPI specification + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= 1 passed, 2 errored in 1.00s ========================= diff --git a/test/__snapshots__/test_recoverable_errors/test_in_cli[2].raw b/test/__snapshots__/test_recoverable_errors/test_in_cli[2].raw new file mode 100644 index 0000000000..91b10c0569 --- /dev/null +++ b/test/__snapshots__/test_recoverable_errors/test_in_cli[2].raw @@ -0,0 +1,43 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.yaml +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 2 +Collected API operations: 2 + +.E + +==================================== ERRORS ==================================== +__________________________________ POST /bar ___________________________________ +Schema Error + +Unresolvable JSON pointer in the schema + +Error details: + JSON pointer: 'components/UnknownParameter' + This typically means that the schema is referencing a component that doesn't exist. + +Ensure that the definition complies with the OpenAPI specification +_____________________________________ /foo _____________________________________ +Schema Error + +Unresolvable JSON pointer in the schema + +Error details: + JSON pointer: 'components/UnknownMethods' + This typically means that the schema is referencing a component that doesn't exist. + +Ensure that the definition complies with the OpenAPI specification + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= 1 passed, 2 errored in 1.00s ========================= diff --git a/test/__snapshots__/test_serialization/test_in_cli[Open API 3.0].raw b/test/__snapshots__/test_serialization/test_in_cli[Open API 3.0].raw new file mode 100644 index 0000000000..863d1da52a --- /dev/null +++ b/test/__snapshots__/test_serialization/test_in_cli[Open API 3.0].raw @@ -0,0 +1,29 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +POST /api/csv E [100%] + +==================================== ERRORS ==================================== +________________________________ POST /api/csv _________________________________ +Runtime Error + +Schemathesis can't serialize data to any of the defined media types: text/csv +You can register your own serializer with `schemathesis.serializer` and Schemathesis will be able to make API calls with this media type. +See https://schemathesis.readthedocs.io/en/stable/how.html#payload-serialization for more information. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-1].raw b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-1].raw new file mode 100644 index 0000000000..4c7a34da38 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-1].raw @@ -0,0 +1,36 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1:1/api +Specification version: Swagger 2.0 +Workers: 1 +Collected API operations: 2 + +GET /api/failure E [ 50%] +GET /api/success E [100%] + +==================================== ERRORS ==================================== +_______________________________ GET /api/failure _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused +_______________________________ GET /api/success _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 2 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-2].raw b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-2].raw new file mode 100644 index 0000000000..76160ce2c2 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 2.0-2].raw @@ -0,0 +1,35 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1:1/api +Specification version: Swagger 2.0 +Workers: 2 +Collected API operations: 2 + +EE + +==================================== ERRORS ==================================== +_______________________________ GET /api/failure _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused +_______________________________ GET /api/success _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 2 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-1].raw b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-1].raw new file mode 100644 index 0000000000..8e1e19e0d4 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-1].raw @@ -0,0 +1,36 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1:1/api +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 2 + +GET /api/failure E [ 50%] +GET /api/success E [100%] + +==================================== ERRORS ==================================== +_______________________________ GET /api/failure _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused +_______________________________ GET /api/success _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 2 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-2].raw b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-2].raw new file mode 100644 index 0000000000..32fd9bc49d --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_connection_error[Open API 3.0-2].raw @@ -0,0 +1,35 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1:1/api +Specification version: Open API 3.0.2 +Workers: 2 +Collected API operations: 2 + +EE + +==================================== ERRORS ==================================== +_______________________________ GET /api/failure _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused +_______________________________ GET /api/success _______________________________ +Network Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 2 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 2.0].raw b/test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 2.0].raw new file mode 100644 index 0000000000..e8a082aa9e --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 2.0].raw @@ -0,0 +1,31 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /api/users E [100%] + +==================================== ERRORS ==================================== +________________________________ GET /api/users ________________________________ +Schema Error + +GET requests should not contain body parameters. + +Tip: Bypass validation using `--validate-schema=false`. Caution: May cause unexpected errors. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 3.0].raw b/test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 3.0].raw new file mode 100644 index 0000000000..e8a082aa9e --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_get_request_with_body[Open API 3.0].raw @@ -0,0 +1,31 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /api/users E [100%] + +==================================== ERRORS ==================================== +________________________________ GET /api/users ________________________________ +Schema Error + +GET requests should not contain body parameters. + +Tip: Bypass validation using `--validate-schema=false`. Caution: May cause unexpected errors. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-1].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-1].raw new file mode 100644 index 0000000000..8646965679 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-1].raw @@ -0,0 +1,31 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Swagger 2.0 +Workers: 1 +Collected API operations: 2 + +GET /api/slow E [ 50%] +GET /api/success . [100%] + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-2].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-2].raw new file mode 100644 index 0000000000..b9d20d041f --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 2.0-2].raw @@ -0,0 +1,30 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Swagger 2.0 +Workers: 2 +Collected API operations: 2 + +?? + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-1].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-1].raw new file mode 100644 index 0000000000..5c9a344f7f --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-1].raw @@ -0,0 +1,31 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 2 + +GET /api/slow E [ 50%] +GET /api/success . [100%] + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-2].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-2].raw new file mode 100644 index 0000000000..0c63aadcc4 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[real-Open API 3.0-2].raw @@ -0,0 +1,30 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Open API 3.0.2 +Workers: 2 +Collected API operations: 2 + +?? + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-1].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-1].raw new file mode 100644 index 0000000000..6693f1ae5d --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-1].raw @@ -0,0 +1,31 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 1 +Collected API operations: 2 + +GET /api/slow E [ 50%] +GET /api/success . [100%] + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-2].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-2].raw new file mode 100644 index 0000000000..a699726042 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 2.0-2].raw @@ -0,0 +1,30 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 2 +Collected API operations: 2 + +?? + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-1].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-1].raw new file mode 100644 index 0000000000..6693f1ae5d --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-1].raw @@ -0,0 +1,31 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 1 +Collected API operations: 2 + +GET /api/slow E [ 50%] +GET /api/success . [100%] + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-2].raw b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-2].raw new file mode 100644 index 0000000000..a699726042 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_hypothesis_failed_event[wsgi-Open API 3.0-2].raw @@ -0,0 +1,30 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 2 +Collected API operations: 2 + +?? + +==================================== ERRORS ==================================== +________________________________ GET /api/slow _________________________________ +Runtime Error + +API response time is too slow! It took 500.00ms, which exceeds the deadline of 20.00ms. + +Tip: Adjust the deadline using `--hypothesis-deadline=MILLIS` or disable with `--hypothesis-deadline=None`. + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +Performed checks: + not_a_server_error N / N passed PASSED + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.0.2-The provided definition doesn't match any of the expected formats or types.].raw b/test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.0.2-The provided definition doesn't match any of the expected formats or types.].raw new file mode 100644 index 0000000000..63db5d51af --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.0.2-The provided definition doesn't match any of the expected formats or types.].raw @@ -0,0 +1,47 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +POST /data E [100%] + +==================================== ERRORS ==================================== +__________________________________ POST /data __________________________________ +Schema Error + +Invalid `bearerAuth` definition + +Location: + components -> securitySchemes -> bearerAuth + +Problematic definition: +{ + "scheme": "bearer", + "bearerFormat": "uuid" +} + +Error details: + The provided definition doesn't match any of the expected formats or types. + +Ensure that the definition complies with the OpenAPI specification + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Experimental Features: + - OpenAPI 3.1: Support for response validation + Feedback: https://github.com/schemathesis/schemathesis/discussions/1822 + +Your feedback is crucial for experimental features. Please visit the provided URL(s) to share your thoughts. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.1.0-'type' is a required property].raw b/test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.1.0-'type' is a required property].raw new file mode 100644 index 0000000000..b0065416b5 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_invalid_schema_with_disabled_validation[3.1.0-'type' is a required property].raw @@ -0,0 +1,47 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: file:/// +Specification version: Open API 3.1.0 +Workers: 1 +Collected API operations: 1 + +POST /data E [100%] + +==================================== ERRORS ==================================== +__________________________________ POST /data __________________________________ +Schema Error + +Invalid `bearerAuth` definition + +Location: + components -> securitySchemes -> bearerAuth + +Problematic definition: +{ + "scheme": "bearer", + "bearerFormat": "uuid" +} + +Error details: + 'type' is a required property + +Ensure that the definition complies with the OpenAPI specification + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Experimental Features: + - OpenAPI 3.1: Support for response validation + Feedback: https://github.com/schemathesis/schemathesis/discussions/1822 + +Your feedback is crucial for experimental features. Please visit the provided URL(s) to share your thoughts. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-None].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-None].raw new file mode 100644 index 0000000000..f60851f154 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-None].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /foo E [100%] + +==================================== ERRORS ==================================== +___________________________________ GET /foo ___________________________________ +Schema Error + +Can not generate data for cookie parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-http127.0.0.1apiv2].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-http127.0.0.1apiv2].raw new file mode 100644 index 0000000000..2f22c233d7 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[cookie-http127.0.0.1apiv2].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: http://127.0.0.1/apiv2 +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /apiv2/foo E [100%] + +==================================== ERRORS ==================================== +________________________________ GET /apiv2/foo ________________________________ +Schema Error + +Can not generate data for cookie parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-None].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-None].raw new file mode 100644 index 0000000000..c90fa1f6d1 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-None].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /foo E [100%] + +==================================== ERRORS ==================================== +___________________________________ GET /foo ___________________________________ +Schema Error + +Can not generate data for header parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-http127.0.0.1apiv2].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-http127.0.0.1apiv2].raw new file mode 100644 index 0000000000..bddbd01bf9 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[header-http127.0.0.1apiv2].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: http://127.0.0.1/apiv2 +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /apiv2/foo E [100%] + +==================================== ERRORS ==================================== +________________________________ GET /apiv2/foo ________________________________ +Schema Error + +Can not generate data for header parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-None].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-None].raw new file mode 100644 index 0000000000..cadb3d8554 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-None].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /foo E [100%] + +==================================== ERRORS ==================================== +___________________________________ GET /foo ___________________________________ +Schema Error + +Can not generate data for path parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-http127.0.0.1apiv2].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-http127.0.0.1apiv2].raw new file mode 100644 index 0000000000..cc639d61b5 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[path-http127.0.0.1apiv2].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: http://127.0.0.1/apiv2 +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /apiv2/foo E [100%] + +==================================== ERRORS ==================================== +________________________________ GET /apiv2/foo ________________________________ +Schema Error + +Can not generate data for path parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-None].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-None].raw new file mode 100644 index 0000000000..a89cf9e050 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-None].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /foo E [100%] + +==================================== ERRORS ==================================== +___________________________________ GET /foo ___________________________________ +Schema Error + +Can not generate data for query parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-http127.0.0.1apiv2].raw b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-http127.0.0.1apiv2].raw new file mode 100644 index 0000000000..be60cd3172 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_missing_content_and_schema[query-http127.0.0.1apiv2].raw @@ -0,0 +1,27 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: http://127.0.0.1/apiv2 +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +GET /apiv2/foo E [100%] + +==================================== ERRORS ==================================== +________________________________ GET /apiv2/foo ________________________________ +Schema Error + +Can not generate data for query parameter "X-Foo"! It should have either `schema` or `content` keywords defined + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_schema_not_available[1].raw b/test/cli/__snapshots__/test_commands/test_schema_not_available[1].raw new file mode 100644 index 0000000000..458272bbc9 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_schema_not_available[1].raw @@ -0,0 +1,10 @@ +Exit code: 1 +--- +Stdout: +Schema Loading Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused + +Tip: Use `--wait-for-schema=NUM` to wait up to NUM seconds for schema availability. diff --git a/test/cli/__snapshots__/test_commands/test_schema_not_available[2].raw b/test/cli/__snapshots__/test_commands/test_schema_not_available[2].raw new file mode 100644 index 0000000000..458272bbc9 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_schema_not_available[2].raw @@ -0,0 +1,10 @@ +Exit code: 1 +--- +Stdout: +Schema Loading Error + +Connection failed + + Failed to establish a new connection: [Error NUM] Connection refused + +Tip: Use `--wait-for-schema=NUM` to wait up to NUM seconds for schema availability. diff --git a/test/cli/__snapshots__/test_commands/test_unresolvable_reference_with_disabled_validation.raw b/test/cli/__snapshots__/test_commands/test_unresolvable_reference_with_disabled_validation.raw new file mode 100644 index 0000000000..44a64e518c --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unresolvable_reference_with_disabled_validation.raw @@ -0,0 +1,44 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.json +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 2 + +GET /bar . [ 50%] +POST /bar E [100%] + +==================================== ERRORS ==================================== +__________________________________ POST /bar ___________________________________ +Schema Error + +Unresolvable JSON pointer in the schema + +Error details: + JSON pointer: 'components/UnknownParameter' + This typically means that the schema is referencing a component that doesn't exist. + +Ensure that the definition complies with the OpenAPI specification +_____________________________________ /foo _____________________________________ +Schema Error + +Unresolvable JSON pointer in the schema + +Error details: + JSON pointer: 'components/UnknownMethods' + This typically means that the schema is referencing a component that doesn't exist. + +Ensure that the definition complies with the OpenAPI specification + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= 1 passed, 2 errored in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-1].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-1].raw new file mode 100644 index 0000000000..caacd03e5a --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-1].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Swagger 2.0 +Workers: 1 +Collected API operations: 1 + +POST /api/unsatisfiable E [100%] + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-2].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-2].raw new file mode 100644 index 0000000000..cd6fd71f8e --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 2.0-2].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Swagger 2.0 +Workers: 2 +Collected API operations: 1 + +E + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-1].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-1].raw new file mode 100644 index 0000000000..89d46d6b37 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-1].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 1 + +POST /api/unsatisfiable E [100%] + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-2].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-2].raw new file mode 100644 index 0000000000..20bb11e2d2 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[real-Open API 3.0-2].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: http://127.0.0.1/schema.yaml +Base URL: http://127.0.0.1/api +Specification version: Open API 3.0.2 +Workers: 2 +Collected API operations: 1 + +E + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-1].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-1].raw new file mode 100644 index 0000000000..c02694ddb1 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-1].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 1 +Collected API operations: 1 + +POST /api/unsatisfiable E [100%] + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-2].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-2].raw new file mode 100644 index 0000000000..683378e19a --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 2.0-2].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 2 +Collected API operations: 1 + +E + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-1].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-1].raw new file mode 100644 index 0000000000..c02694ddb1 --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-1].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 1 +Collected API operations: 1 + +POST /api/unsatisfiable E [100%] + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-2].raw b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-2].raw new file mode 100644 index 0000000000..683378e19a --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsatisfiable[wsgi-Open API 3.0-2].raw @@ -0,0 +1,34 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: /schema.yaml +Base URL: /api +Specification version: Swagger 2.0 +Workers: 2 +Collected API operations: 1 + +E + +==================================== ERRORS ==================================== +___________________________ POST /api/unsatisfiable ____________________________ +Schema Error + +Failed to generate test cases for this API operation. Possible reasons: + + - Contradictory schema constraints, such as a minimum value exceeding the maximum. + - Excessive schema complexity, which hinders parameter generation. + +Tip: Examine the schema for inconsistencies and consider simplifying it. + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +============================== 1 errored in 1.00s ============================== diff --git a/test/cli/__snapshots__/test_commands/test_unsupported_regex.raw b/test/cli/__snapshots__/test_commands/test_unsupported_regex.raw new file mode 100644 index 0000000000..5ae8108dfc --- /dev/null +++ b/test/cli/__snapshots__/test_commands/test_unsupported_regex.raw @@ -0,0 +1,32 @@ +Exit code: 1 +--- +Stdout: +======================= Schemathesis test session starts ======================= +Schema location: file:///tmp/schema.yaml +Base URL: file:/// +Specification version: Open API 3.0.2 +Workers: 1 +Collected API operations: 2 + +POST /bar . [ 50%] +POST /foo E [100%] + +==================================== ERRORS ==================================== +__________________________________ POST /foo ___________________________________ +Schema Error + +Invalid regular expression. Pattern `\\p{Alpha}` is not recognized - `bad escape \p at position 0` + +Tip: Ensure your regex is compatible with Python's syntax. For guidance, visit: https://docs.python.org/3/library/re.html + +Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks + +Need more help? + Join our Discord server: https://discord.gg/R9ASRAmHnA +=================================== SUMMARY ==================================== + +No checks were performed. + +Hint: You can visualize test results in Schemathesis.io by using `--report` in your CLI command. + +========================= 1 passed, 1 errored in 1.00s ========================= diff --git a/test/cli/__snapshots__/test_commands/test_wait_for_schema_not_enough.raw b/test/cli/__snapshots__/test_commands/test_wait_for_schema_not_enough.raw index 893f13b05c..75d8b9d741 100644 --- a/test/cli/__snapshots__/test_commands/test_wait_for_schema_not_enough.raw +++ b/test/cli/__snapshots__/test_commands/test_wait_for_schema_not_enough.raw @@ -5,4 +5,4 @@ Schema Loading Error Connection failed - Failed to establish a new connection: [Errno 111] Connection refused + Failed to establish a new connection: [Error NUM] Connection refused diff --git a/test/cli/output/test_default.py b/test/cli/output/test_default.py index 02383acb57..78f0e03d99 100644 --- a/test/cli/output/test_default.py +++ b/test/cli/output/test_default.py @@ -1,6 +1,5 @@ import io import os -import sys import click import hypothesis @@ -288,44 +287,6 @@ def test_after_execution_attributes(execution_context, after_execution): assert execution_context.current_line_length == 2 -@pytest.mark.parametrize("show_errors_tracebacks", (True, False)) -def test_display_single_error(capsys, swagger_20, operation, execution_context, show_errors_tracebacks): - # Given exception is multiline - exception = None - try: - exec("some invalid code") - except SyntaxError as exc: - exception = exc - - result = models.TestResult( - operation.method, - operation.path, - verbose_name=f"{operation.method} {operation.full_path}", - data_generation_method=[DataGenerationMethod.default()], - ) - result.add_error(exception) - # When the related test result is displayed - execution_context.show_errors_tracebacks = show_errors_tracebacks - default.display_single_error(execution_context, SerializedTestResult.from_test_result(result)) - lines = capsys.readouterr().out.strip().split("\n") - # Then it should be correctly formatted and displayed in red color - if sys.version_info >= (3, 10): - expected = 'File "", line 1\n some invalid code\n ^^^^^^^\nSyntaxError: invalid syntax' - elif sys.version_info <= (3, 8): - expected = 'File "", line 1\n some invalid code\n ^\nSyntaxError: invalid syntax' - else: - expected = 'File "", line 1\n some invalid code\n ^\nSyntaxError: invalid syntax' - if show_errors_tracebacks: - lines = click.unstyle("\n".join(lines)).split("\n") - assert lines[1] == "Traceback (most recent call last):" - # There is a path on the next line, it is simpler to not check it since it doesn't give much value - # But presence of traceback itself is checked - expected = f' exec("some invalid code")\n {expected}' - assert "\n".join(lines[3:8]) == expected.strip("\n") - else: - assert "\n".join(lines[1:6]) == strip_style_win32(click.style(expected, fg="red")).rstrip("\n") - - @pytest.mark.parametrize("verbosity", (0, 1)) def test_display_failures(swagger_20, capsys, execution_context, results_set, verbosity, response): del response.request.headers["Content-Type"] @@ -354,38 +315,6 @@ def test_display_failures(swagger_20, capsys, execution_context, results_set, ve assert "curl -X GET http://127.0.0.1:8080/api/failure" in out -@pytest.mark.parametrize("show_errors_tracebacks", (True, False)) -def test_display_errors(swagger_20, capsys, results_set, execution_context, show_errors_tracebacks): - # Given two test results - success and error - operation = models.APIOperation("/api/error", "GET", {}, swagger_20) - error = models.TestResult( - operation.method, - operation.full_path, - verbose_name=f"{operation.method} {operation.full_path}", - data_generation_method=[DataGenerationMethod.default()], - seed=123, - ) - error.add_error(ConnectionError("Connection refused!")) - results_set.append(error) - execution_context.results.append(SerializedTestResult.from_test_result(error)) - event = Finished.from_results(results_set, 1.0) - # When the errors are displayed - execution_context.show_errors_tracebacks = show_errors_tracebacks - default.display_errors(execution_context, event) - out = capsys.readouterr().out.strip() - # Then section title is displayed - assert " ERRORS " in out - help_message_exists = ( - "Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks" in out - ) - # And help message is displayed only if tracebacks are not shown - assert help_message_exists is not show_errors_tracebacks - # And operation with an error is displayed as a subsection - assert " GET /v1/api/error " in out - # And the error itself is displayed - assert "ConnectionError: Connection refused!" in out - - @pytest.mark.parametrize("show_errors_tracebacks", (True, False)) def test_display_internal_error(capsys, execution_context, show_errors_tracebacks): execution_context.show_errors_tracebacks = show_errors_tracebacks diff --git a/test/cli/test_commands.py b/test/cli/test_commands.py index 0d05c88854..d5f3db0631 100644 --- a/test/cli/test_commands.py +++ b/test/cli/test_commands.py @@ -157,27 +157,11 @@ def test_auth_and_authorization_header_are_disallowed(cli, schema_url, header, s @pytest.mark.parametrize("workers", (1, 2)) -def test_schema_not_available(cli, workers): +def test_schema_not_available(cli, workers, snapshot_cli): # When the given schema is unreachable - result = cli.run("http://127.0.0.1:1/schema.yaml", f"--workers={workers}") # Then the whole Schemathesis run should fail - assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout # And error message is displayed - if platform.system() == "Windows": - detail = "[WinError 10061] No connection could be made because the target machine actively refused it" - else: - detail = "[Errno 111] Connection refused" - assert ( - result.stdout - == f"""Schema Loading Error - -Connection failed - - Failed to establish a new connection: {detail} - -Tip: Use `--wait-for-schema=NUM` to wait up to NUM seconds for schema availability. -""" - ) + assert cli.run("http://127.0.0.1:1/schema.yaml", f"--workers={workers}") == snapshot_cli def test_schema_not_available_wsgi(cli, loadable_flask_app, snapshot_cli): @@ -579,25 +563,12 @@ def test_execute_missing_schema(cli, openapi3_base_url, url, message, workers): @pytest.mark.operations("success", "slow") @pytest.mark.parametrize("workers", (1, 2)) -def test_hypothesis_failed_event(cli, cli_args, workers): +@pytest.mark.snapshot(replace_multi_worker_progress="??", replace_statistic=True) +def test_hypothesis_failed_event(cli, cli_args, workers, snapshot_cli): # When the Hypothesis deadline option is set manually, and it is smaller than the response time - result = cli.run(*cli_args, "--hypothesis-deadline=20", f"--workers={workers}") # Then the whole Schemathesis run should fail - assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout - # And the given operation should be displayed as an error - lines = result.stdout.split("\n") - if workers == 1: - assert lines[7].startswith("GET /api/slow E") - else: - # It could be in any sequence, because of multiple threads - assert lines[7].split("\n")[0] in ("E.", ".E", "EE") - # empty line after all tests progress output - assert lines[8] == "" # And the proper error message should be displayed - assert "DeadlineExceeded: API response time is too slow! " in result.stdout - assert "which exceeds the deadline of 20.00ms" in result.stdout - # And the CLI should not suggest showing full tracebacks to the user - assert "Add this option to your command line parameters to see full tracebacks" not in result.stdout, result.stdout + assert cli.run(*cli_args, "--hypothesis-deadline=20", f"--workers={workers}") == snapshot_cli @pytest.mark.operations("success", "slow") @@ -647,24 +618,12 @@ def test_seed(cli, cli_args, workers): @pytest.mark.operations("unsatisfiable") @pytest.mark.parametrize("workers", (1, 2)) -def test_unsatisfiable(cli, cli_args, workers): +def test_unsatisfiable(cli, cli_args, workers, snapshot_cli): # When the app's schema contains parameters that can't be generated # For example if it contains contradiction in the parameters' definition - requires to be integer AND string at the # same time - result = cli.run(*cli_args, f"--workers={workers}") - # Then the whole Schemathesis run should fail - assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout - # And standard Hypothesis error should not appear in the output - assert "You can add @seed" not in result.stdout - # And this operation should be marked as errored in the progress line - lines = result.stdout.split("\n") - if workers == 1: - assert lines[7].startswith("POST /api/unsatisfiable E") - else: - assert lines[7] == "E" # And more clear error message is displayed instead of Hypothesis one - lines = result.stdout.split("\n") - assert "hypothesis.errors.Unsatisfiable: Unable to satisfy schema parameters for this API operation" in lines + assert cli.run(*cli_args, f"--workers={workers}") == snapshot_cli @pytest.mark.operations("flaky") @@ -791,26 +750,11 @@ def test_multiple_failures_different_check(cli, schema_url): @pytest.mark.parametrize("workers", (1, 2)) -def test_connection_error(cli, schema_url, workers): +def test_connection_error(cli, schema_url, workers, snapshot_cli): # When the given base_url is unreachable - result = cli.run(schema_url, "--base-url=http://127.0.0.1:1/api", f"--workers={workers}") # Then the whole Schemathesis run should fail - assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout - # And all collected API operations should be marked as errored - lines = result.stdout.split("\n") - if workers == 1: - assert lines[7].startswith("GET /api/failure E") - assert lines[8].startswith("GET /api/success E") - else: - assert lines[7] == "EE" - # And errors section title should be displayed - assert "= ERRORS =" in result.stdout - # And all API operations should be mentioned in this section as subsections - assert "_ GET /api/success _" in result.stdout - assert "_ GET /api/failure _" in result.stdout # And the proper error messages should be displayed for each operation - assert "Max retries exceeded with url: /api/success" in result.stdout - assert "Max retries exceeded with url: /api/failure" in result.stdout + assert cli.run(schema_url, "--base-url=http://127.0.0.1:1/api", f"--workers={workers}") == snapshot_cli @pytest.fixture @@ -1536,18 +1480,17 @@ def test_openapi_links_multiple_threads(cli, cli_args, schema_url, recursion_lim assert lines[7] == expected + "." if hypothesis_max_examples else expected -def test_get_request_with_body(testdir, cli, base_url, hypothesis_max_examples, schema_with_get_payload): +def test_get_request_with_body(testdir, cli, base_url, hypothesis_max_examples, schema_with_get_payload, snapshot_cli): schema_file = testdir.makefile(".yaml", schema=yaml.dump(schema_with_get_payload)) - result = cli.run( - str(schema_file), - f"--base-url={base_url}", - f"--hypothesis-max-examples={hypothesis_max_examples or 1}", - "--show-errors-tracebacks", - "--validate-schema=true", + assert ( + cli.run( + str(schema_file), + f"--base-url={base_url}", + f"--hypothesis-max-examples={hypothesis_max_examples or 1}", + "--validate-schema=true", + ) + == snapshot_cli ) - assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout - lines = result.stdout.splitlines() - assert "OperationSchemaError: Body parameters are defined for GET request." in lines @pytest.mark.operations("slow") @@ -1651,7 +1594,7 @@ def test_reserved_characters_in_operation_name(testdir, empty_open_api_3_schema) assert "GET /foo:bar . [100%]" in result.outlines -def test_unsupported_regex(testdir, cli, empty_open_api_3_schema): +def test_unsupported_regex(testdir, cli, empty_open_api_3_schema, snapshot_cli): def make_definition(min_items): return { "post": { @@ -1681,18 +1624,9 @@ def make_definition(min_items): "/bar": make_definition(min_items=0), } schema_file = testdir.makefile(".yaml", schema=yaml.dump(empty_open_api_3_schema)) - result = cli.run(str(schema_file), "--dry-run", "--hypothesis-max-examples=1") # Then if it is possible it should generate at least something - assert "POST /bar ." in result.stdout # And if it is not then there should be an error with a descriptive error message - assert "POST /foo E" in result.stdout - lines = result.stdout.splitlines() - for idx, line in enumerate(lines): # noqa: B007 - if "__ POST /foo __" in line: - break - else: - pytest.fail("Line not found") - assert r"Got pattern='\\p{Alpha}', but this is not valid syntax for a Python regular expression" in lines[idx + 1] + assert cli.run(str(schema_file), "--dry-run", "--hypothesis-max-examples=1") == snapshot_cli @pytest.mark.parametrize("extra", ("--auth='test:wrong'", "-H Authorization: Basic J3Rlc3Q6d3Jvbmcn")) @@ -1869,7 +1803,7 @@ def test_get_exit_code(swagger_20, capsys): @pytest.mark.parametrize("base_url", (None, "http://127.0.0.1/apiv2")) @pytest.mark.parametrize("location", ("path", "query", "header", "cookie")) -def test_missing_content_and_schema(cli, base_url, tmp_path, testdir, empty_open_api_3_schema, location): +def test_missing_content_and_schema(cli, base_url, tmp_path, testdir, empty_open_api_3_schema, location, snapshot_cli): debug_file = tmp_path / "debug.jsonl" # When an Open API 3 parameter is missing `schema` & `content` empty_open_api_3_schema["paths"] = { @@ -1885,19 +1819,9 @@ def test_missing_content_and_schema(cli, base_url, tmp_path, testdir, empty_open ] if base_url is not None: args.append(f"--base-url={base_url}") - result = cli.run(*args) - lines = result.stdout.split("\n") # Then CLI should show that this API operation errored # And show the proper message under its "ERRORS" section - if base_url is None: - assert lines[7].startswith("GET /foo E") - else: - assert lines[7].startswith("GET /apiv2/foo E") - assert "_ GET /apiv2/foo _" in lines[10] - assert ( - lines[11] == f'OperationSchemaError: Can not generate data for {location} parameter "X-Foo"! ' - "It should have either `schema` or `content` keywords defined" - ) + assert cli.run(*args) == snapshot_cli # And emitted Before / After event pairs have the same correlation ids with debug_file.open(encoding="utf-8") as fd: events = [json.loads(line) for line in fd] @@ -2092,52 +2016,24 @@ def test_disable_report_suggestion(monkeypatch, cli, schema_url): ), ) def test_invalid_schema_with_disabled_validation( - testdir, cli, openapi_3_schema_with_invalid_security, version, details + testdir, cli, openapi_3_schema_with_invalid_security, version, details, snapshot_cli ): # When there is an error in the schema openapi_3_schema_with_invalid_security["openapi"] = version schema_file = testdir.makefile(".json", schema=json.dumps(openapi_3_schema_with_invalid_security)) # And the validation is disabled (default) - result = cli.run(str(schema_file), "--dry-run", "--experimental=openapi-3.1") - assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout # Then we should show an error message derived from JSON Schema - assert ( - f"""OperationSchemaError: Invalid `bearerAuth` definition - -Location: - components -> securitySchemes -> bearerAuth - -Problematic definition: -{{ - "scheme": "bearer", - "bearerFormat": "uuid" -}} - -Error details: - {details} + assert cli.run(str(schema_file), "--dry-run", "--experimental=openapi-3.1") == snapshot_cli -Ensure that the definition complies with the OpenAPI specification""" - in result.stdout - ) - -def test_unresolvable_reference_with_disabled_validation(testdir, cli, open_api_3_schema_with_recoverable_errors): +def test_unresolvable_reference_with_disabled_validation( + testdir, cli, open_api_3_schema_with_recoverable_errors, snapshot_cli +): # When there is an error in the schema schema_file = testdir.makefile(".json", schema=json.dumps(open_api_3_schema_with_recoverable_errors)) # And the validation is disabled (default) - result = cli.run(str(schema_file), "--dry-run") - assert result.exit_code == ExitCode.TESTS_FAILED, result.stdout # Then we should show an error message derived from JSON Schema - assert ( - """OperationSchemaError: Unresolvable JSON pointer in the schema - -Error details: - JSON pointer: 'components/UnknownMethods' - This typically means that the schema is referencing a component that doesn't exist. - -Ensure that the definition complies with the OpenAPI specification""" - in result.stdout - ) + assert cli.run(str(schema_file), "--dry-run") == snapshot_cli @pytest.mark.parametrize("value", ("true", "false")) diff --git a/test/cli/test_junitxml.py b/test/cli/test_junitxml.py index d2953c4096..b4522fe5a2 100644 --- a/test/cli/test_junitxml.py +++ b/test/cli/test_junitxml.py @@ -64,4 +64,4 @@ def test_junitxml_file(cli, schema_url, hypothesis_max_examples, tmp_path): assert testcases[2].attrib["name"] == "POST /api/unsatisfiable" assert testcases[2][0].tag == "error" assert testcases[2][0].attrib["type"] == "error" - assert "Unable to satisfy schema parameters for this API operation" in testcases[2][0].attrib["message"] + assert "Failed to generate test cases for this API operation" in testcases[2][0].attrib["message"] diff --git a/test/conftest.py b/test/conftest.py index 94780cedbf..6f4ae2a0fb 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -7,7 +7,7 @@ from pathlib import Path from textwrap import dedent from types import SimpleNamespace -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union import pytest import requests @@ -113,6 +113,7 @@ def pytest_generate_tests(metafunc): def pytest_configure(config): config.addinivalue_line("markers", "operations(*names): Add only specified API operations to the test application.") config.addinivalue_line("markers", "service(**kwargs): Setup mock server for Schemathesis.io.") + config.addinivalue_line("markers", "snapshot(**kwargs): Configure snapshot tests.") config.addinivalue_line("markers", "hypothesis_nested: Mark tests with nested Hypothesis tests.") config.addinivalue_line( "markers", @@ -271,6 +272,9 @@ class CliSnapshotConfig: replace_schema_location: bool = True replace_tmp_dir: bool = True replace_duration: bool = True + replace_multi_worker_progress: Union[bool, str] = True + replace_statistic: bool = False + replace_error_codes: bool = True @classmethod def from_request(cls, request: FixtureRequest) -> "CliSnapshotConfig": @@ -295,11 +299,28 @@ def serialize(self, data: str) -> str: if self.replace_tmp_dir: with keep_cwd(): data = data.replace(str(self.testdir.tmpdir) + os.path.sep, "/tmp/") + if self.replace_multi_worker_progress: + lines = data.splitlines() + for idx, line in enumerate(lines): + if re.match(r"^[.FSE]+$", line): + if isinstance(self.replace_multi_worker_progress, str): + lines[idx] = self.replace_multi_worker_progress + else: + lines[idx] = "".join(sorted(line)) + data = "\n".join(lines) + "\n" + if self.replace_statistic: + data = re.sub("[0-9]+ / [0-9]+ passed", "N / N passed", data) + data = re.sub("([0-9]+ passed,? )|([0-9]+ errored,? )", "", data) + if self.replace_error_codes: + data = data.replace("Errno 111", "Error NUM").replace("WinError 10061", "Error NUM") + data = data.replace( + "No connection could be made because the target machine actively refused it", "Connection refused" + ) if self.replace_duration: + data = re.sub(r"It took [0-9]+\.[0-9]{2}ms", "It took 500.00ms", data) lines = data.splitlines() lines[-1] = re.sub(r"in [0-9]+\.[0-9]{2}s", "in 1.00s", lines[-1]) - data = "\n".join(lines) - data += "\n" + data = "\n".join(lines) + "\n" return data diff --git a/test/runner/test_runner.py b/test/runner/test_runner.py index 12229635f5..267f3350a4 100644 --- a/test/runner/test_runner.py +++ b/test/runner/test_runner.py @@ -763,7 +763,7 @@ def test_unsatisfiable_example(empty_open_api_3_schema): ).execute() # And the tests are failing because of the unsatisfiable schema assert finished.has_errors - assert "Unable to satisfy schema parameters for this API operation" in after.result.errors[0].exception + assert "Failed to generate test cases for this API operation" in after.result.errors[0].exception @pytest.mark.operations("success") diff --git a/test/test_hypothesis.py b/test/test_hypothesis.py index 1c78e73bb7..2079e96819 100644 --- a/test/test_hypothesis.py +++ b/test/test_hypothesis.py @@ -117,7 +117,7 @@ def test_invalid_body_in_get(swagger_20): ] ), ) - with pytest.raises(OperationSchemaError, match=r"^Body parameters are defined for GET request.$"): + with pytest.raises(OperationSchemaError, match=r"^GET requests should not contain body parameters.$"): get_case_strategy(operation).example() diff --git a/test/test_lazy.py b/test/test_lazy.py index 7f8507e7ba..c05aea97b9 100644 --- a/test/test_lazy.py +++ b/test/test_lazy.py @@ -389,7 +389,7 @@ def test_(request, case): ) result = testdir.runpytest("-v") result.assert_outcomes(passed=1, failed=1) - result.stdout.re_match_lines([r"E +OperationSchemaError: Body parameters are defined for GET request."]) + result.stdout.re_match_lines([r"E +BodyInGetRequestError: GET requests should not contain body parameters."]) @pytest.mark.parametrize( diff --git a/test/test_parameters.py b/test/test_parameters.py index 9919c62e3a..fbba171f09 100644 --- a/test/test_parameters.py +++ b/test/test_parameters.py @@ -424,7 +424,7 @@ def test_(case): ) # Then an error should be propagated with a relevant error message result = testdir.run_and_assert(failed=1) - result.stdout.re_match_lines([r"E +OperationSchemaError: Body parameters are defined for GET request."]) + result.stdout.re_match_lines([r"E +BodyInGetRequestError: GET requests should not contain body parameters."]) def test_json_media_type(testdir): diff --git a/test/test_recoverable_errors.py b/test/test_recoverable_errors.py index c5625ba51e..e4018b0c9d 100644 --- a/test/test_recoverable_errors.py +++ b/test/test_recoverable_errors.py @@ -100,22 +100,11 @@ def test_(case): @pytest.mark.parametrize("workers", (1, 2)) -def test_in_cli(testdir, cli, open_api_3_schema_with_recoverable_errors, workers): +def test_in_cli(testdir, cli, open_api_3_schema_with_recoverable_errors, workers, snapshot_cli): schema_file = testdir.makefile(".yaml", schema=yaml.dump(open_api_3_schema_with_recoverable_errors)) - result = cli.run(str(schema_file), "--dry-run", "--show-errors-tracebacks", f"--workers={workers}") - lines = result.stdout.splitlines() # Then valid operation should be tested # And errors on the single operation error should be displayed - if workers == 1: - assert lines[7].startswith("GET /bar .") - assert lines[8].startswith("POST /bar E") - else: - assert lines[7] in ("E.", ".E") - error = " JSON pointer: 'components/UnknownParameter'" - assert len([line for line in lines if error in line]) == 1 - assert "1 passed, 2 errored" in lines[-1] - assert "____ /foo ____" in result.stdout - assert " JSON pointer: 'components/UnknownMethods'" in result.stdout + assert cli.run(str(schema_file), "--dry-run", f"--workers={workers}") == snapshot_cli def test_direct_access(schema): diff --git a/test/test_serialization.py b/test/test_serialization.py index c5120839fb..e84df73c35 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -115,6 +115,12 @@ def test(case): test() +@pytest.mark.openapi_version("3.0") +@pytest.mark.operations("csv_payload") +def test_in_cli(cli, schema_url, snapshot_cli): + assert cli.run(schema_url) == snapshot_cli + + @pytest.mark.parametrize("method", ("as_requests_kwargs", "as_werkzeug_kwargs")) def test_serialize_yaml(open_api_3_schema_with_yaml_payload, method): # See GH-1010