chore: Clarify CLI error messages for issues coming from network and …

…configuration

Ref: #1607 #1835

docs/changelog.rst

@@ -13,6 +13,7 @@ Changelog
 - Clarify CLI error messages for loading hooks and WSGI applications.
 - Clarify CLI option docstrings.
 - Provide an error message if an internal error happened inside CLI event handler.
+- Clarify CLI error messages for issues coming from network and configuration. `#1607`_, `#1835`_
 
 .. _v3.20.2:
 
@@ -3508,6 +3509,7 @@ Deprecated
 .. _0.3.0: https://github.com/schemathesis/schemathesis/compare/v0.2.0...v0.3.0
 .. _0.2.0: https://github.com/schemathesis/schemathesis/compare/v0.1.0...v0.2.0
 
+.. _#1835: https://github.com/schemathesis/schemathesis/issues/1835
 .. _#1820: https://github.com/schemathesis/schemathesis/issues/1820
 .. _#1819: https://github.com/schemathesis/schemathesis/issues/1819
 .. _#1808: https://github.com/schemathesis/schemathesis/issues/1808
@@ -3542,6 +3544,7 @@ Deprecated
 .. _#1627: https://github.com/schemathesis/schemathesis/issues/1627
 .. _#1625: https://github.com/schemathesis/schemathesis/issues/1625
 .. _#1614: https://github.com/schemathesis/schemathesis/issues/1614
+.. _#1607: https://github.com/schemathesis/schemathesis/issues/1607
 .. _#1602: https://github.com/schemathesis/schemathesis/issues/1602
 .. _#1592: https://github.com/schemathesis/schemathesis/issues/1592
 .. _#1591: https://github.com/schemathesis/schemathesis/issues/1591

src/schemathesis/cli/output/default.py

@@ -20,6 +20,7 @@
     FALSE_VALUES,
     ISSUE_TRACKER_URL,
 )
+from ...exceptions import RuntimeErrorType
 from ...experimental import GLOBAL_EXPERIMENTS
 from ...models import Response, Status
 from ...runner import events
@@ -130,15 +131,22 @@ def display_errors(context: ExecutionContext, event: events.Finished) -> None:
 
     display_section_name("ERRORS")
     should_display_full_traceback_message = False
-    for result in context.results:
+    if context.workers_num > 1:
+        # Events may come out of order when multiple workers are involved
+        # Sort them to get a stable output
+        results = sorted(context.results, key=lambda r: r.verbose_name)
+    else:
+        results = context.results
+    for result in results:
         if not result.has_errors:
             continue
         should_display_full_traceback_message |= display_single_error(context, result)
     if event.generic_errors:
         display_generic_errors(context, event.generic_errors)
     if should_display_full_traceback_message and not context.show_errors_tracebacks:
         click.secho(
-            "Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks", fg="red"
+            "\nAdd this option to your command line parameters to see full tracebacks: --show-errors-tracebacks",
+            fg="red",
         )
     click.secho(
         f"\nNeed more help?\n" f"    Join our Discord server: {DISCORD_LINK}",
@@ -160,23 +168,58 @@ def display_generic_errors(context: ExecutionContext, errors: List[SerializedErr
         _display_error(context, error)
 
 
-def display_full_traceback_message(exception: str) -> bool:
+def display_full_traceback_message(error: SerializedError) -> bool:
     # Some errors should not trigger the message that suggests to show full tracebacks to the user
-    return not exception.startswith(("DeadlineExceeded", "OperationSchemaError"))
+    return not error.exception.startswith(
+        ("DeadlineExceeded", "OperationSchemaError", "requests.exceptions", "SerializationNotPossible")
+    )
+
+
+def bold(option: str) -> str:
+    return click.style(option, bold=True)
+
+
+DISABLE_SSL_SUGGESTION = f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}."
+DISABLE_SCHEMA_VALIDATION_SUGGESTION = (
+    f"Bypass validation using {bold('`--validate-schema=false`')}. " f"Caution: May cause unexpected errors."
+)
+
+RUNTIME_ERROR_SUGGESTIONS = {
+    RuntimeErrorType.CONNECTION_SSL: DISABLE_SSL_SUGGESTION,
+    RuntimeErrorType.HYPOTHESIS_DEADLINE_EXCEEDED: (
+        f"Adjust the deadline using {bold('`--hypothesis-deadline=MILLIS`')} or "
+        f"disable with {bold('`--hypothesis-deadline=None`')}."
+    ),
+    RuntimeErrorType.HYPOTHESIS_UNSATISFIABLE: "Examine the schema for inconsistencies and consider simplifying it.",
+    RuntimeErrorType.SCHEMA_BODY_IN_GET_REQUEST: DISABLE_SCHEMA_VALIDATION_SUGGESTION,
+    RuntimeErrorType.SCHEMA_INVALID_REGULAR_EXPRESSION: "Ensure your regex is compatible with Python's syntax. "
+    "For guidance, visit: https://docs.python.org/3/library/re.html",
+}
 
 
 def _display_error(context: ExecutionContext, error: SerializedError) -> bool:
-    if context.show_errors_tracebacks:
-        message = error.exception_with_traceback
+    if error.title:
+        if error.type == RuntimeErrorType.SCHEMA_GENERIC:
+            click.secho("Schema Error", fg="red", bold=True)
+        else:
+            click.secho(error.title, fg="red", bold=True)
+        click.echo()
+        if error.message:
+            click.echo(error.message)
+    elif error.message:
+        click.echo(error.message)
     else:
-        message = error.exception
-    if error.exception.startswith("DeadlineExceeded"):
-        message += (
-            "Consider extending the deadline with the `--hypothesis-deadline` CLI option.\n"
-            "You can disable it completely with `--hypothesis-deadline=None`.\n"
-        )
-    click.secho(message, fg="red")
-    return display_full_traceback_message(error.exception)
+        click.echo(error.exception)
+    if error.extras:
+        extras = error.extras
+    elif context.show_errors_tracebacks:
+        extras = _split_traceback(error.exception_with_traceback)
+    else:
+        extras = []
+    _display_extras(extras)
+    suggestion = RUNTIME_ERROR_SUGGESTIONS.get(error.type)
+    _maybe_display_tip(suggestion)
+    return display_full_traceback_message(error)
 
 
 def display_failures(context: ExecutionContext, event: events.Finished) -> None:
@@ -480,13 +523,9 @@ def display_check_result(check_name: str, results: Dict[Union[str, Status], int]
 VERIFY_URL_SUGGESTION = "Verify that the URL points directly to the Open API schema"
 
 
-def bold(option: str) -> str:
-    return click.style(option, bold=True)
-
-
 SCHEMA_ERROR_SUGGESTIONS = {
     # SSL-specific connection issue
-    SchemaErrorType.CONNECTION_SSL: f"Bypass SSL verification with {bold('`--request-tls-verify=false`')}.",
+    SchemaErrorType.CONNECTION_SSL: DISABLE_SSL_SUGGESTION,
     # Other connection problems
     SchemaErrorType.CONNECTION_OTHER: f"Use {bold('`--wait-for-schema=NUM`')} to wait up to NUM seconds for schema availability.",
     # Response issues
@@ -496,7 +535,7 @@ def bold(option: str) -> str:
     # OpenAPI specification issues
     SchemaErrorType.OPEN_API_UNSPECIFIED_VERSION: f"Include the version in the schema or manually set it with {bold('`--force-schema-version`')}.",
     SchemaErrorType.OPEN_API_UNSUPPORTED_VERSION: f"Proceed with {bold('`--force-schema-version`')}. Caution: May not be fully supported.",
-    SchemaErrorType.OPEN_API_INVALID_SCHEMA: f"Bypass validation using {bold('`--validate-schema=false`')}. Caution: May cause unexpected errors.",
+    SchemaErrorType.OPEN_API_INVALID_SCHEMA: DISABLE_SCHEMA_VALIDATION_SUGGESTION,
     # YAML specific issues
     SchemaErrorType.YAML_NUMERIC_STATUS_CODES: "Convert numeric status codes to strings.",
     SchemaErrorType.YAML_NON_STRING_KEYS: "Convert non-string keys to strings.",
@@ -509,20 +548,34 @@ def should_skip_suggestion(context: ExecutionContext, event: events.InternalErro
     return event.subtype == SchemaErrorType.CONNECTION_OTHER and context.wait_for_schema is not None
 
 
+def _split_traceback(traceback: str) -> List[str]:
+    return [entry for entry in traceback.splitlines() if entry]
+
+
+def _display_extras(extras: List[str]) -> None:
+    if extras:
+        click.echo()
+    for extra in extras:
+        click.secho(f"    {extra}")
+
+
+def _maybe_display_tip(suggestion: Optional[str]) -> None:
+    # Display suggestion if any
+    if suggestion is not None:
+        click.secho(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}")
+
+
 def display_internal_error(context: ExecutionContext, event: events.InternalError) -> None:
     click.secho(event.title, fg="red", bold=True)
     click.echo()
     click.secho(event.message)
     if event.type == InternalErrorType.SCHEMA:
         extras = event.extras
     elif context.show_errors_tracebacks:
-        extras = [entry for entry in event.exception_with_traceback.splitlines() if entry]
+        extras = _split_traceback(event.exception_with_traceback)
     else:
         extras = [event.exception]
-    if extras:
-        click.echo()
-    for extra in extras:
-        click.secho(f"    {extra}")
+    _display_extras(extras)
     if not should_skip_suggestion(context, event):
         if event.type == InternalErrorType.SCHEMA and isinstance(event.subtype, SchemaErrorType):
             suggestion = SCHEMA_ERROR_SUGGESTIONS.get(event.subtype)
@@ -532,9 +585,7 @@ def display_internal_error(context: ExecutionContext, event: events.InternalErro
             )
         else:
             suggestion = f"To see full tracebacks, add {bold('`--show-errors-tracebacks`')} to your CLI options"
-        # Display suggestion if any
-        if suggestion is not None:
-            click.secho(f"\n{click.style('Tip:', bold=True, fg='green')} {suggestion}")
+        _maybe_display_tip(suggestion)
 
 
 def handle_initialized(context: ExecutionContext, event: events.Initialized) -> None:

src/schemathesis/exceptions.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import enum
 import json
+import re
 import traceback
 from dataclasses import dataclass, field
 from hashlib import sha1
@@ -15,6 +16,7 @@
     import hypothesis.errors
     from jsonschema import RefResolutionError, ValidationError
     from .transports.responses import GenericResponse
+    from requests import RequestException
 
 
 class CheckFailed(AssertionError):
@@ -206,6 +208,22 @@ def actual_test(*args: Any, **kwargs: Any) -> NoReturn:
         return actual_test
 
 
+@dataclass
+class BodyInGetRequestError(OperationSchemaError):
+    __module__ = "builtins"
+
+
+class InvalidRegularExpression(OperationSchemaError):
+    __module__ = "builtins"
+
+    @classmethod
+    def from_hypothesis_jsonschema_message(cls, message: str) -> "InvalidRegularExpression":
+        match = re.search(r"pattern='(.*?)'.*?\((.*?)\)", message)
+        if match:
+            message = f"Invalid regular expression. Pattern `{match.group(1)}` is not recognized - `{match.group(2)}`"
+        return cls(message)
+
+
 def truncated_json(data: Any, max_lines: int = 10, max_width: int = 80) -> str:
     # Convert JSON to string with indentation
     indent = 4
@@ -241,7 +259,26 @@ def from_exc(cls, exc: hypothesis.errors.DeadlineExceeded) -> "DeadlineExceeded"
 
 
 @enum.unique
-class SchemaErrorType(enum.Enum):
+class RuntimeErrorType(str, enum.Enum):
+    # Connection related issues
+    CONNECTION_SSL = "connection_ssl"
+    CONNECTION_OTHER = "connection_other"
+    NETWORK_OTHER = "network_other"
+
+    # Hypothesis issues
+    HYPOTHESIS_DEADLINE_EXCEEDED = "hypothesis_deadline_exceeded"
+    HYPOTHESIS_UNSATISFIABLE = "hypothesis_unsatisfiable"
+
+    SCHEMA_BODY_IN_GET_REQUEST = "schema_body_in_get_request"
+    SCHEMA_INVALID_REGULAR_EXPRESSION = "schema_invalid_regular_expression"
+    SCHEMA_GENERIC = "schema_generic"
+
+    # Unclassified
+    UNCLASSIFIED = "unclassified"
+
+
+@enum.unique
+class SchemaErrorType(str, enum.Enum):
     # Connection related issues
     CONNECTION_SSL = "connection_ssl"
     CONNECTION_OTHER = "connection_other"
@@ -352,10 +389,6 @@ def for_media_type(cls, media_type: str) -> "SerializationNotPossible":
         return cls(SERIALIZATION_FOR_TYPE_IS_NOT_POSSIBLE_MESSAGE.format(media_type))
 
 
-class InvalidRegularExpression(Exception):
-    __module__ = "builtins"
-
-
 class UsageError(Exception):
     """Incorrect usage of Schemathesis functions."""
 
@@ -384,3 +417,24 @@ def extract_nth_traceback(trace: Optional[TracebackType], n: int) -> Optional[Tr
         trace = trace.tb_next
         depth += 1
     return trace
+
+
+def remove_ssl_line_number(text: str) -> str:
+    return re.sub(r"\(_ssl\.c:\d+\)", "", text)
+
+
+def extract_requests_exception_details(exc: RequestException) -> Tuple[str, List[str]]:
+    from requests.exceptions import SSLError, ConnectionError
+
+    if isinstance(exc, SSLError):
+        message = "SSL verification problem"
+        reason = str(exc.args[0].reason)
+        extra = [remove_ssl_line_number(reason)]
+    elif isinstance(exc, ConnectionError):
+        message = "Connection failed"
+        _, reason = exc.args[0].reason.args[0].split(":", maxsplit=1)
+        extra = [reason.strip()]
+    else:
+        message = "Network problem"
+        extra = []
+    return message, extra

src/schemathesis/loaders.py

@@ -4,7 +4,7 @@
 from functools import lru_cache
 from typing import Callable, TypeVar, cast, TYPE_CHECKING, TextIO, Any, Dict, Type, BinaryIO
 
-from .exceptions import SchemaError, SchemaErrorType
+from .exceptions import SchemaError, SchemaErrorType, extract_requests_exception_details
 
 if TYPE_CHECKING:
     from .transports.responses import GenericResponse
@@ -13,10 +13,6 @@
 R = TypeVar("R", bound="GenericResponse")
 
 
-def remove_ssl_line_number(text: str) -> str:
-    return re.sub(r"\(_ssl\.c:\d+\)", "", text)
-
-
 def load_schema_from_url(loader: Callable[[], R]) -> R:
     import requests
 
@@ -25,20 +21,13 @@ def load_schema_from_url(loader: Callable[[], R]) -> R:
     except requests.RequestException as exc:
         request = cast(requests.PreparedRequest, exc.request)
         if isinstance(exc, requests.exceptions.SSLError):
-            message = "SSL verification problem"
             type_ = SchemaErrorType.CONNECTION_SSL
-            reason = str(exc.args[0].reason)
-            extra = [remove_ssl_line_number(reason)]
         elif isinstance(exc, requests.exceptions.ConnectionError):
-            message = "Connection failed"
             type_ = SchemaErrorType.CONNECTION_OTHER
-            _, reason = exc.args[0].reason.args[0].split(":", maxsplit=1)
-            extra = [reason.strip()]
         else:
-            message = "Network problem"
             type_ = SchemaErrorType.NETWORK_OTHER
-            extra = []
-        raise SchemaError(message=message, type=type_, url=request.url, response=exc.response, extras=extra) from exc
+        message, extras = extract_requests_exception_details(exc)
+        raise SchemaError(message=message, type=type_, url=request.url, response=exc.response, extras=extras) from exc
     _raise_for_status(response)
     return response