From a880f24ef0ff450adc976159bbefdd81c83bd9ca Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Sun, 3 May 2020 14:57:10 +0200 Subject: [PATCH] feat: Third-party compatibility fixups mechanism --- docs/changelog.rst | 2 ++ docs/compatibility.rst | 30 +++++++++++++++++++ docs/faq.rst | 18 +++++++++++ docs/index.rst | 1 + src/schemathesis/__init__.py | 2 +- src/schemathesis/cli/__init__.py | 11 ++++++- src/schemathesis/converter.py | 36 +++------------------- src/schemathesis/fixups/__init__.py | 25 ++++++++++++++++ src/schemathesis/fixups/fast_api.py | 30 +++++++++++++++++++ src/schemathesis/hooks.py | 12 ++++++++ src/schemathesis/loaders.py | 2 ++ src/schemathesis/runner/__init__.py | 10 +++++++ src/schemathesis/utils.py | 37 ++++++++++++++++++++++- test/cli/test_commands.py | 19 ++++++++++-- test/conftest.py | 24 +++++++++++++++ test/test_fixups.py | 46 +++++++++++++++++++++++++++++ 16 files changed, 268 insertions(+), 37 deletions(-) create mode 100644 docs/compatibility.rst create mode 100644 src/schemathesis/fixups/__init__.py create mode 100644 src/schemathesis/fixups/fast_api.py create mode 100644 test/test_fixups.py diff --git a/docs/changelog.rst b/docs/changelog.rst index a9d8ace7af..55f2627458 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Added for hook functions that do not accept this argument. - A new hook system that allows generic hook dispatching. It comes with new hook locations. For more details see "Customization" section in our documentation. - New ``before_process_path`` hook. +- Third-party compatibility fixups mechanism. Currently there is one fixup for `FastAPI `_. `#503`_ Deprecated ~~~~~~~~~~ @@ -1008,6 +1009,7 @@ Fixed .. _#519: https://github.com/kiwicom/schemathesis/issues/519 .. _#513: https://github.com/kiwicom/schemathesis/issues/513 .. _#504: https://github.com/kiwicom/schemathesis/issues/504 +.. _#503: https://github.com/kiwicom/schemathesis/issues/503 .. _#499: https://github.com/kiwicom/schemathesis/issues/499 .. _#497: https://github.com/kiwicom/schemathesis/issues/497 .. _#496: https://github.com/kiwicom/schemathesis/issues/496 diff --git a/docs/compatibility.rst b/docs/compatibility.rst new file mode 100644 index 0000000000..302757a957 --- /dev/null +++ b/docs/compatibility.rst @@ -0,0 +1,30 @@ +.. _compatibility: + +Compatibility +============= + +By default, Schemathesis is strict on Open API spec interpretation, but other 3rd-party tools often are more flexible +and not always comply with the spec. + +FastAPI +------- + +`FastAPI `_ uses `pydantic `_ for JSON Schema +generation and it produces Draft 7 compatible schemas. But Open API 2 / 3.0.x use earlier versions of JSON Schema (Draft 4 and Wright Draft 00 respectively) which leads +to incompatibilities when Schemathesis parses input schema. + +This is a `known issue `_ on the FastAPI side +and Schemathesis provides a way to handle such schemas. The idea is based on converting Draft 7 keywords syntax to Draft 4. + +To use it you need to add this code before you load your schema with Schemathesis: + +.. code:: python + + import schemathesis + + # will install all available compatibility fixups. + schemathesis.fixups.install() + # You can also provide a list of fixup names as the first argument + # schemathesis.fixups.install(["fastapi"]) + +If you use CLI, then you can utilize ``--fixups=all`` option. diff --git a/docs/faq.rst b/docs/faq.rst index 8ab5739429..bb24ad1b5d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -54,6 +54,24 @@ The ``case`` object, that is injected in each test can be modified, assuming you def test_api(case): case.path_parameters["user_id"] = 42 +Why does Schemathesis fail to parse my API schema generate by FastAPI? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because FastAPI uses JSON Draft 7 under the hood (via ``pydantic``) which is not compatible with JSON drafts defined by +the Open API 2 / 3.0.x versions. It is a `known issue `_ on the FastAPI side. +Schemathesis is more strict in schema handling by default, but we provide optional fixups for this case: + +.. code:: python + + import schemathesis + + # will install all available compatibility fixups. + schemathesis.fixups.install() + # You can also provide a list of fixup names as the first argument + # schemathesis.fixups.install(["fastapi"]) + +For more information, take a look into the "Compatibility" section. + Working with API schemas ------------------------ diff --git a/docs/index.rst b/docs/index.rst index e267d8e831..aab82fe1e8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,7 @@ Welcome to schemathesis's documentation! :caption: Contents: usage + compatibility customization targeted faq diff --git a/src/schemathesis/__init__.py b/src/schemathesis/__init__.py index 0bb0b3cb1f..65338555cb 100644 --- a/src/schemathesis/__init__.py +++ b/src/schemathesis/__init__.py @@ -1,4 +1,4 @@ -from . import hooks +from . import fixups, hooks from ._hypothesis import init_default_strategies, register_string_format from .cli import register_check from .constants import __version__ diff --git a/src/schemathesis/cli/__init__.py b/src/schemathesis/cli/__init__.py index fa1d23a661..1ae56ab55e 100644 --- a/src/schemathesis/cli/__init__.py +++ b/src/schemathesis/cli/__init__.py @@ -10,6 +10,7 @@ from .. import checks as checks_module from .. import models, runner +from ..fixups import ALL_FIXUPS from ..runner import events from ..runner.targeted import DEFAULT_TARGETS_NAMES, Target from ..types import Filter @@ -151,7 +152,13 @@ def schemathesis(pre_run: Optional[str] = None) -> None: is_eager=True, default=False, ) -@click.option("--store-network-log", help="Store requests and responses into a file", type=click.File("w")) +@click.option("--store-network-log", help="Store requests and responses into a file.", type=click.File("w")) +@click.option( + "--fixups", + help="Install specified compatibility fixups.", + multiple=True, + type=click.Choice(list(ALL_FIXUPS) + ["all"]), +) @click.option( "--hypothesis-deadline", help="Duration in milliseconds that each individual example with a test is not allowed to exceed.", @@ -199,6 +206,7 @@ def run( # pylint: disable=too-many-arguments junit_xml: Optional[click.utils.LazyFile] = None, show_errors_tracebacks: bool = False, store_network_log: Optional[click.utils.LazyFile] = None, + fixups: Tuple[str] = (), # type: ignore hypothesis_deadline: Optional[Union[int, NotSet]] = None, hypothesis_derandomize: Optional[bool] = None, hypothesis_max_examples: Optional[int] = None, @@ -238,6 +246,7 @@ def run( # pylint: disable=too-many-arguments targets=selected_targets, workers_num=workers_num, validate_schema=validate_schema, + fixups=fixups, hypothesis_deadline=hypothesis_deadline, hypothesis_derandomize=hypothesis_derandomize, hypothesis_max_examples=hypothesis_max_examples, diff --git a/src/schemathesis/converter.py b/src/schemathesis/converter.py index d753d938ab..53f397cf36 100644 --- a/src/schemathesis/converter.py +++ b/src/schemathesis/converter.py @@ -1,5 +1,7 @@ from copy import deepcopy -from typing import Any, Dict, List, Union, overload +from typing import Any, Dict + +from .utils import traverse_schema def to_json_schema(schema: Dict[str, Any], nullable_name: str) -> Dict[str, Any]: @@ -25,40 +27,10 @@ def to_json_schema(schema: Dict[str, Any], nullable_name: str) -> Dict[str, Any] return schema -Schema = Union[Dict[str, Any], List, str, float, int] - - -@overload def to_json_schema_recursive(schema: Dict[str, Any], nullable_name: str) -> Dict[str, Any]: - pass - - -@overload -def to_json_schema_recursive(schema: List, nullable_name: str) -> List: - pass - - -@overload -def to_json_schema_recursive(schema: str, nullable_name: str) -> str: - pass - - -@overload -def to_json_schema_recursive(schema: float, nullable_name: str) -> float: - pass - - -def to_json_schema_recursive(schema: Schema, nullable_name: str) -> Schema: """Apply ``to_json_schema`` recursively. This version is needed for cases where the input schema was not resolved and ``to_json_schema`` wasn't applied recursively. """ - if isinstance(schema, dict): - schema = to_json_schema(schema, nullable_name) - for key, sub_item in schema.items(): - schema[key] = to_json_schema_recursive(sub_item, nullable_name) - elif isinstance(schema, list): - for idx, sub_item in enumerate(schema): - schema[idx] = to_json_schema_recursive(sub_item, nullable_name) - return schema + return traverse_schema(schema, to_json_schema, nullable_name) diff --git a/src/schemathesis/fixups/__init__.py b/src/schemathesis/fixups/__init__.py new file mode 100644 index 0000000000..a7cc94fb1a --- /dev/null +++ b/src/schemathesis/fixups/__init__.py @@ -0,0 +1,25 @@ +from typing import Iterable, List, Optional + +from . import fast_api + +ALL_FIXUPS = {"fast_api": fast_api} + + +def install(fixups: Optional[Iterable[str]] = None) -> None: + """Install fixups. + + Without the first argument installs all available fixups. + """ + fixups = fixups or list(ALL_FIXUPS.keys()) + for name in fixups: + ALL_FIXUPS[name].install() # type: ignore + + +def uninstall(fixups: Optional[Iterable[str]] = None) -> None: + """Uninstall fixups. + + Without the first argument uninstalls all available fixups. + """ + fixups = fixups or list(ALL_FIXUPS.keys()) + for name in fixups: + ALL_FIXUPS[name].uninstall() # type: ignore diff --git a/src/schemathesis/fixups/fast_api.py b/src/schemathesis/fixups/fast_api.py new file mode 100644 index 0000000000..26a2e8d675 --- /dev/null +++ b/src/schemathesis/fixups/fast_api.py @@ -0,0 +1,30 @@ +from typing import Any, Dict + +from ..hooks import HookContext, register, unregister +from ..utils import traverse_schema + + +def install() -> None: + register(before_load_schema) + + +def uninstall() -> None: + unregister(before_load_schema) + + +def before_load_schema(context: HookContext, schema: Dict[str, Any]) -> None: + traverse_schema(schema, _handle_boundaries) + + +def _handle_boundaries(schema: Dict[str, Any]) -> Dict[str, Any]: + """Convert Draft 7 keywords to Draft 4 compatible versions. + + FastAPI uses ``pydantic``, which generates Draft 7 compatible schemas. + """ + for boundary_name, boundary_exclusive_name in (("maximum", "exclusiveMaximum"), ("minimum", "exclusiveMinimum")): + value = schema.get(boundary_exclusive_name) + # `bool` check is needed, since in Python `True` is an instance of `int` + if isinstance(value, (int, float)) and not isinstance(value, bool): + schema[boundary_exclusive_name] = True + schema[boundary_name] = value + return schema diff --git a/src/schemathesis/hooks.py b/src/schemathesis/hooks.py index ae42987884..c16e38e067 100644 --- a/src/schemathesis/hooks.py +++ b/src/schemathesis/hooks.py @@ -131,6 +131,12 @@ def dispatch(self, name: str, context: HookContext, *args: Any, **kwargs: Any) - for hook in self.get_hooks(name): hook(context, *args, **kwargs) + def unregister(self, hook: Callable) -> None: + """Unregister a specific hook.""" + # It removes this function from all places + for hooks in self._hooks.values(): + hooks[:] = [item for item in hooks if item is not hook] + def unregister_all(self) -> None: """Remove all registered hooks. @@ -174,9 +180,15 @@ def before_process_path(context: HookContext, path: str, methods: Dict[str, Any] pass +@HookDispatcher.register_spec +def before_load_schema(context: HookContext, raw_schema: Dict[str, Any]) -> None: + pass + + GLOBAL_HOOK_DISPATCHER = HookDispatcher() dispatch = GLOBAL_HOOK_DISPATCHER.dispatch get_hooks = GLOBAL_HOOK_DISPATCHER.get_hooks +unregister = GLOBAL_HOOK_DISPATCHER.unregister unregister_all = GLOBAL_HOOK_DISPATCHER.unregister_all diff --git a/src/schemathesis/loaders.py b/src/schemathesis/loaders.py index 6f8f893835..4fe9af3a8f 100644 --- a/src/schemathesis/loaders.py +++ b/src/schemathesis/loaders.py @@ -12,6 +12,7 @@ from . import spec_schemas from .constants import USER_AGENT from .exceptions import HTTPError +from .hooks import HookContext, dispatch from .lazy import LazySchema from .schemas import BaseSchema, OpenApi30, SwaggerV20 from .types import Filter, PathLike @@ -115,6 +116,7 @@ def from_dict( validate_schema: bool = True, ) -> BaseSchema: """Get a proper abstraction for the given raw schema.""" + dispatch("before_load_schema", HookContext(), raw_schema) if "swagger" in raw_schema: _maybe_validate_schema(raw_schema, spec_schemas.SWAGGER_20, validate_schema) return SwaggerV20( diff --git a/src/schemathesis/runner/__init__.py b/src/schemathesis/runner/__init__.py index 7708d70dc3..3f6c1daf33 100644 --- a/src/schemathesis/runner/__init__.py +++ b/src/schemathesis/runner/__init__.py @@ -3,6 +3,7 @@ import hypothesis.errors +from .. import fixups as _fixups from .. import loaders from ..checks import DEFAULT_CHECKS from ..models import CheckFunction @@ -24,6 +25,7 @@ def prepare( # pylint: disable=too-many-arguments seed: Optional[int] = None, exit_first: bool = False, store_interactions: bool = False, + fixups: Iterable[str] = (), # Schema loading loader: Callable = loaders.from_uri, base_url: Optional[str] = None, @@ -82,6 +84,7 @@ def prepare( # pylint: disable=too-many-arguments headers=headers, request_timeout=request_timeout, store_interactions=store_interactions, + fixups=fixups, ) @@ -125,6 +128,7 @@ def execute_from_schema( seed: Optional[int] = None, exit_first: bool = False, store_interactions: bool = False, + fixups: Iterable[str] = (), ) -> Generator[events.ExecutionEvent, None, None]: """Execute tests for the given schema. @@ -132,6 +136,12 @@ def execute_from_schema( """ # pylint: disable=too-many-locals try: + if fixups: + if "all" in fixups: + _fixups.install() + else: + _fixups.install(fixups) + if app is not None: app = import_app(app) schema = load_schema( diff --git a/src/schemathesis/utils.py b/src/schemathesis/utils.py index 85cf12ff01..b84d776bad 100644 --- a/src/schemathesis/utils.py +++ b/src/schemathesis/utils.py @@ -6,7 +6,7 @@ import warnings from contextlib import contextmanager from functools import wraps -from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Type, Union +from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Type, Union, overload from urllib.parse import urlsplit, urlunsplit import requests @@ -196,3 +196,38 @@ def import_app(path: str) -> Any: # may return a parent module (system dependent) module = sys.modules[path] return getattr(module, name) + + +Schema = Union[Dict[str, Any], List, str, float, int] + + +@overload +def traverse_schema(schema: Dict[str, Any], callback: Callable, *args: Any, **kwargs: Any) -> Dict[str, Any]: + pass + + +@overload +def traverse_schema(schema: List, callback: Callable, *args: Any, **kwargs: Any) -> List: + pass + + +@overload +def traverse_schema(schema: str, callback: Callable, *args: Any, **kwargs: Any) -> str: + pass + + +@overload +def traverse_schema(schema: float, callback: Callable, *args: Any, **kwargs: Any) -> float: + pass + + +def traverse_schema(schema: Schema, callback: Callable[..., Dict[str, Any]], *args: Any, **kwargs: Any) -> Schema: + """Apply callback recursively to the given schema.""" + if isinstance(schema, dict): + schema = callback(schema, *args, **kwargs) + for key, sub_item in schema.items(): + schema[key] = traverse_schema(sub_item, callback, *args, **kwargs) + elif isinstance(schema, list): + for idx, sub_item in enumerate(schema): + schema[idx] = traverse_schema(sub_item, callback, *args, **kwargs) + return schema diff --git a/test/cli/test_commands.py b/test/cli/test_commands.py index 15fa4f6ad9..1fea034ea5 100644 --- a/test/cli/test_commands.py +++ b/test/cli/test_commands.py @@ -12,7 +12,7 @@ from _pytest.main import ExitCode from hypothesis import HealthCheck, Phase, Verbosity -from schemathesis import Case +from schemathesis import Case, fixups from schemathesis._compat import metadata from schemathesis.checks import ALL_CHECKS from schemathesis.cli import reset_checks @@ -191,7 +191,8 @@ def test_commands_run_help(cli): " path.", "", " --show-errors-tracebacks Show full tracebacks for internal errors.", - " --store-network-log FILENAME Store requests and responses into a file", + " --store-network-log FILENAME Store requests and responses into a file.", + " --fixups [fast_api|all] Install specified compatibility fixups.", " --hypothesis-deadline INTEGER RANGE", " Duration in milliseconds that each individual", " example with a test is not allowed to exceed.", @@ -276,6 +277,7 @@ def test_execute_arguments(cli, mocker, simple_schema, args, expected): "hypothesis_options": {}, "workers_num": 1, "exit_first": False, + "fixups": (), "auth": None, "auth_type": None, "headers": {}, @@ -1112,3 +1114,16 @@ def test_chained_internal_exception(testdir, cli, base_url): assert result.exit_code == ExitCode.TESTS_FAILED lines = result.stdout.splitlines() assert "The above exception was the direct cause of the following exception:" in lines + + +@pytest.fixture() +def fast_api_fixup(): + yield + fixups.uninstall() + + +def test_fast_api_fixup(testdir, cli, base_url, fast_api_schema, fast_api_fixup): + # When schema contains Draft 7 definitions as ones from FastAPI may contain + schema_file = testdir.makefile(".yaml", schema=yaml.dump(fast_api_schema)) + result = cli.run(str(schema_file), f"--base-url={base_url}", "--hypothesis-max-examples=1", "--fixups=all") + assert result.exit_code == ExitCode.OK diff --git a/test/conftest.py b/test/conftest.py index 5fc5d0a95e..df783d0680 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -143,6 +143,30 @@ def simple_openapi(): } +@pytest.fixture(scope="session") +def fast_api_schema(): + # This schema contains definitions from JSON Schema Draft 7 + return { + "openapi": "3.0.2", + "info": {"title": "Test", "description": "Test", "version": "0.1.0"}, + "paths": { + "/query": { + "get": { + "parameters": [ + { + "name": "value", + "in": "query", + "required": True, + "schema": {"type": "integer", "exclusiveMinimum": 0, "exclusiveMaximum": 10}, + }, + ], + "responses": {"200": {"description": "OK"}}, + } + } + }, + } + + @pytest.fixture(scope="session") def swagger_20(simple_schema): return schemathesis.from_dict(simple_schema) diff --git a/test/test_fixups.py b/test/test_fixups.py new file mode 100644 index 0000000000..7c7b3b15b9 --- /dev/null +++ b/test/test_fixups.py @@ -0,0 +1,46 @@ +import pytest + +from schemathesis import fixups + + +def test_global_fixup(testdir, fast_api_schema): + testdir.makepyfile( + """ +import schemathesis +from hypothesis import settings + +schemathesis.fixups.install() +schema = schemathesis.from_dict({schema}) + +def teardown_module(module): + schemathesis.fixups.uninstall() + assert schemathesis.hooks.get_hooks("before_load_schema") == [] + +@schema.parametrize() +@settings(max_examples=1) +def test(case): + assert 0 < case.query["value"] < 10 + """.format( + schema=fast_api_schema + ), + ) + result = testdir.runpytest("-s") + result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + "value, expected", + ( + # No-op case + ({"exclusiveMinimum": True, "minimum": 5}, {"exclusiveMinimum": True, "minimum": 5}), + # Draft 7 to Draft 4 + ({"exclusiveMinimum": 5}, {"exclusiveMinimum": True, "minimum": 5}), + ({"exclusiveMaximum": 5}, {"exclusiveMaximum": True, "maximum": 5}), + # Nested cases + ({"schema": {"exclusiveMaximum": 5}}, {"schema": {"exclusiveMaximum": True, "maximum": 5}}), + ([{"schema": {"exclusiveMaximum": 5}}], [{"schema": {"exclusiveMaximum": True, "maximum": 5}}]), + ), +) +def test_fastapi_schema_conversion(value, expected): + fixups.fast_api.before_load_schema(None, value) + assert value == expected