From cae4563b365c6d2e43f364f462e84b1e598c833f Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Fri, 18 Sep 2020 14:48:31 +0200 Subject: [PATCH] feat: Stateful testing via Open API links for the `pytest` runner --- docs/changelog.rst | 8 + docs/stateful.rst | 35 ++++- src/schemathesis/__init__.py | 1 + src/schemathesis/_hypothesis.py | 9 +- src/schemathesis/cli/__init__.py | 13 +- src/schemathesis/cli/callbacks.py | 7 + src/schemathesis/constants.py | 1 + src/schemathesis/extra/pytest_plugin.py | 85 ++++++++++- src/schemathesis/models.py | 30 +++- src/schemathesis/runner/__init__.py | 7 +- src/schemathesis/runner/impl/__init__.py | 2 +- src/schemathesis/runner/impl/core.py | 53 +------ src/schemathesis/runner/impl/threadpool.py | 11 +- src/schemathesis/schemas.py | 23 ++- src/schemathesis/specs/openapi/_hypothesis.py | 7 +- src/schemathesis/specs/openapi/schemas.py | 12 +- src/schemathesis/stateful.py | 53 ++++++- test/conftest.py | 5 +- test/specs/openapi/test_links.py | 4 +- test/test_stateful.py | 142 ++++++++++++++++++ 20 files changed, 419 insertions(+), 89 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 865ae23f33..685beae793 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,11 +6,18 @@ Changelog `Unreleased`_ ------------- +Added +~~~~~ + +- Stateful testing via Open API links for the ``pytest`` runner. `#616`_ + Changed ~~~~~~~ - Check name is no longer displayed in the CLI output, since its verbose message is already displayed. This change also simplifies the internal structure of the runner events. +- The ``stateful`` argument type in the ``runner.prepare`` is ``Optional[Stateful]`` instead of ``Optional[str]``. Use + ``schemathesis.Stateful`` enum. `2.4.1`_ - 2020-09-17 --------------------- @@ -1368,6 +1375,7 @@ Fixed .. _#621: https://github.com/schemathesis/schemathesis/issues/621 .. _#618: https://github.com/schemathesis/schemathesis/issues/618 .. _#617: https://github.com/schemathesis/schemathesis/issues/617 +.. _#616: https://github.com/schemathesis/schemathesis/issues/616 .. _#614: https://github.com/schemathesis/schemathesis/issues/614 .. _#612: https://github.com/schemathesis/schemathesis/issues/612 .. _#600: https://github.com/schemathesis/schemathesis/issues/600 diff --git a/docs/stateful.rst b/docs/stateful.rst index b1758101e7..46a39de07c 100644 --- a/docs/stateful.rst +++ b/docs/stateful.rst @@ -3,8 +3,8 @@ Stateful testing ================ -By default, Schemathesis generates random data for all endpoints in your schema. With Schemathesis's `stateful testing` -Schemathesis CLI will try to reuse data from requests that were sent and responses received for generating requests to +By default, Schemathesis generates random data for all endpoints in your schema. With Schemathesis's ``stateful testing`` +Schemathesis will try to reuse data from requests that were sent and responses received for generating requests to other endpoints. Open API Links @@ -61,9 +61,11 @@ Based on this definition, Schemathesis will: In this case, it is much more likely that instead of a 404 response for a randomly-generated ``user_id`` we'll receive something else - for example, HTTP codes 200 or 500. -By default, stateful testing is disabled. You can add it via ``--stateful=links`` CLI option. Please, note that more +By default, stateful testing is disabled. You can add it via the ``--stateful=links`` CLI option or with the ``stateful=Stateful.links`` argument to ``parametrize``. Please, note that more different algorithms for stateful testing might be implemented in the future. +CLI: + .. code:: bash schemathesis run --stateful=links http://0.0.0.0/schema.yaml @@ -80,9 +82,32 @@ different algorithms for stateful testing might be implemented in the future. ... +Python tests: + +.. code:: python + + from schemathesis import Stateful + + @schema.parametrize(stateful=Stateful.links) + def test_api(case): + response = case.call() + ... + Each additional test will be indented and prefixed with ``->`` in the CLI output. -You can specify recursive links if you want. The default recursion depth limit is ``5``, it can be changed with -``--stateful-recursion-limit=`` CLI option. +You can specify recursive links if you want. The default recursion depth limit is ``5``, it can be changed with the +``--stateful-recursion-limit=`` CLI option or with the ``stateful_recursion_limit=`` argument to ``parametrize``. + +**NOTE**. If you use stateful testing in Python tests, make sure you use the ``case.call`` method that automatically stores the response for further usage. +Alternatively, you could use ``case.store_response`` and store the received response by hand: + +.. code:: python + + @schema.parametrize(stateful=Stateful.links) + def test_api(case): + response = case.call() # stores the response automatically + # OR, store it manually + response = requests.request(**case.as_requests_kwargs()) + case.store_response(response) Even though this feature appears only in Open API 3.0 specification, under Open API 2.0, you can use it via the ``x-links`` extension, the syntax is the same, but you need to use the ``x-links`` keyword instead of ``links``. diff --git a/src/schemathesis/__init__.py b/src/schemathesis/__init__.py index 65bb0aa6f6..19fe839184 100644 --- a/src/schemathesis/__init__.py +++ b/src/schemathesis/__init__.py @@ -5,5 +5,6 @@ from .models import Case from .specs import graphql from .specs.openapi._hypothesis import init_default_strategies, register_string_format +from .stateful import Stateful init_default_strategies() diff --git a/src/schemathesis/_hypothesis.py b/src/schemathesis/_hypothesis.py index b45cd32aea..6052174759 100644 --- a/src/schemathesis/_hypothesis.py +++ b/src/schemathesis/_hypothesis.py @@ -9,6 +9,7 @@ from .exceptions import InvalidSchema from .hooks import GLOBAL_HOOK_DISPATCHER, HookContext, HookDispatcher from .models import Case, Endpoint +from .stateful import Feedback, Stateful def create_test( @@ -16,7 +17,12 @@ def create_test( ) -> Callable: """Create a Hypothesis test.""" hook_dispatcher = getattr(test, "_schemathesis_hooks", None) - strategy = endpoint.as_strategy(hooks=hook_dispatcher) + feedback: Optional[Feedback] + if endpoint.schema.stateful == Stateful.links: + feedback = Feedback(endpoint.schema.stateful, endpoint) + else: + feedback = None + strategy = endpoint.as_strategy(hooks=hook_dispatcher, feedback=feedback) wrapped_test = hypothesis.given(case=strategy)(test) if seed is not None: wrapped_test = hypothesis.seed(seed)(wrapped_test) @@ -25,6 +31,7 @@ def create_test( setup_default_deadline(wrapped_test) if settings is not None: wrapped_test = settings(wrapped_test) + wrapped_test._schemathesis_feedback = feedback # type: ignore return add_examples(wrapped_test, endpoint, hook_dispatcher=hook_dispatcher) diff --git a/src/schemathesis/cli/__init__.py b/src/schemathesis/cli/__init__.py index a30c7638d1..7719da323f 100644 --- a/src/schemathesis/cli/__init__.py +++ b/src/schemathesis/cli/__init__.py @@ -11,10 +11,12 @@ from .. import checks as checks_module from .. import runner from .. import targets as targets_module +from ..constants import DEFAULT_STATEFUL_RECURSION_LIMIT from ..fixups import ALL_FIXUPS from ..hooks import GLOBAL_HOOK_DISPATCHER, HookContext, HookDispatcher, HookScope from ..models import CheckFunction -from ..runner import DEFAULT_STATEFUL_RECURSION_LIMIT, events +from ..runner import events +from ..stateful import Stateful from ..targets import Target from ..types import Filter from . import callbacks, cassettes, output @@ -198,7 +200,12 @@ def schemathesis(pre_run: Optional[str] = None) -> None: multiple=True, type=click.Choice(list(ALL_FIXUPS) + ["all"]), ) -@click.option("--stateful", help="Utilize stateful testing capabilities.", type=click.Choice(["links"])) +@click.option( + "--stateful", + help="Utilize stateful testing capabilities.", + type=click.Choice([item.name for item in Stateful]), + callback=callbacks.convert_stateful, +) @click.option( "--stateful-recursion-limit", help="Limit recursion depth for stateful testing.", @@ -255,7 +262,7 @@ def run( # pylint: disable=too-many-arguments show_errors_tracebacks: bool = False, store_network_log: Optional[click.utils.LazyFile] = None, fixups: Tuple[str] = (), # type: ignore - stateful: Optional[str] = None, + stateful: Optional[Stateful] = None, stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT, hypothesis_deadline: Optional[Union[int, NotSet]] = None, hypothesis_derandomize: Optional[bool] = None, diff --git a/src/schemathesis/cli/callbacks.py b/src/schemathesis/cli/callbacks.py index 840f1207e1..e8ecd22cfb 100644 --- a/src/schemathesis/cli/callbacks.py +++ b/src/schemathesis/cli/callbacks.py @@ -8,6 +8,7 @@ from requests import PreparedRequest, RequestException from .. import utils +from ..stateful import Stateful def validate_schema(ctx: click.core.Context, param: click.core.Parameter, raw_value: str) -> str: @@ -110,6 +111,12 @@ def convert_verbosity( return hypothesis.Verbosity[value] +def convert_stateful(ctx: click.core.Context, param: click.core.Parameter, value: Optional[str]) -> Optional[Stateful]: + if value is None: + return value + return Stateful[value] + + @contextmanager def reraise_format_error(raw_value: str) -> Generator[None, None, None]: try: diff --git a/src/schemathesis/constants.py b/src/schemathesis/constants.py index 4cc23aeea0..946c17e981 100644 --- a/src/schemathesis/constants.py +++ b/src/schemathesis/constants.py @@ -9,3 +9,4 @@ USER_AGENT = f"schemathesis/{__version__}" DEFAULT_DEADLINE = 500 # pragma: no mutate +DEFAULT_STATEFUL_RECURSION_LIMIT = 5 # pragma: no mutate diff --git a/src/schemathesis/extra/pytest_plugin.py b/src/schemathesis/extra/pytest_plugin.py index d38a37e0c9..adc6d16467 100644 --- a/src/schemathesis/extra/pytest_plugin.py +++ b/src/schemathesis/extra/pytest_plugin.py @@ -1,5 +1,5 @@ from functools import partial -from typing import Any, Callable, Generator, List, Optional, Type, cast +from typing import Any, Callable, Generator, List, Optional, Type, Union, cast import pytest from _pytest import fixtures, nodes @@ -7,12 +7,15 @@ from _pytest.fixtures import FuncFixtureInfo from _pytest.nodes import Node from _pytest.python import Class, Function, FunctionDefinition, Metafunc, Module, PyCollector +from _pytest.runner import runtestprotocol +from _pytest.warning_types import PytestWarning from hypothesis.errors import InvalidArgument # pylint: disable=ungrouped-imports from packaging import version from .._hypothesis import create_test from ..exceptions import InvalidSchema from ..models import Endpoint +from ..stateful import Feedback from ..utils import is_schemathesis_test USE_FROM_PARENT = version.parse(pytest.__version__) >= version.parse("5.4.0") @@ -53,7 +56,15 @@ def _gen_items(self, endpoint: Endpoint) -> Generator[Function, None, None]: metafunc = self._parametrize(cls, definition, fixtureinfo) if not metafunc._calls: - yield create(SchemathesisFunction, name=name, parent=self.parent, callobj=funcobj, fixtureinfo=fixtureinfo) + yield create( + SchemathesisFunction, + name=name, + parent=self.parent, + callobj=funcobj, + fixtureinfo=fixtureinfo, + test_func=self.test_function, + originalname=self.name, + ) else: fixtures.add_funcarg_pseudo_fixture_def(self.parent, metafunc, fixturemanager) fixtureinfo.prune_dependency_tree() @@ -68,6 +79,7 @@ def _gen_items(self, endpoint: Endpoint) -> Generator[Function, None, None]: fixtureinfo=fixtureinfo, keywords={callspec.id: True}, originalname=name, + test_func=self.test_function, ) def _get_class_parent(self) -> Optional[Type]: @@ -105,7 +117,21 @@ def collect(self) -> List[Function]: # type: ignore pytest.fail("Error during collection") +NOT_USED_STATEFUL_TESTING_MESSAGE = ( + "You are using stateful testing, but no responses were stored during the test! " + "Please, use `case.call` or `case.store_response` in your test to enable stateful tests." +) + + class SchemathesisFunction(Function): # pylint: disable=too-many-ancestors + def __init__( + self, *args: Any, test_func: Callable, test_name: Optional[str] = None, recursion_level: int = 0, **kwargs: Any + ) -> None: + super().__init__(*args, **kwargs) + self.test_function = test_func + self.test_name = test_name + self.recursion_level = recursion_level + def _getobj(self) -> partial: """Tests defined as methods require `self` as the first argument. @@ -113,6 +139,51 @@ def _getobj(self) -> partial: """ return partial(self.obj, self.parent.obj) + @property + def feedback(self) -> Optional[Feedback]: + return getattr(self.obj, "_schemathesis_feedback", None) + + def warn_if_stateful_responses_not_stored(self) -> None: + feedback = self.feedback + if feedback is not None and not feedback.stateful_tests: + self.warn(PytestWarning(NOT_USED_STATEFUL_TESTING_MESSAGE)) + + def _get_stateful_tests(self) -> List["SchemathesisFunction"]: + feedback = self.feedback + recursion_level = self.recursion_level + if feedback is None or recursion_level >= feedback.endpoint.schema.stateful_recursion_limit: + return [] + previous_test_name = self.test_name or f"{feedback.endpoint.method}:{feedback.endpoint.full_path}" + + def make_test( + endpoint: Endpoint, test: Union[Callable, InvalidSchema], previous_tests: str + ) -> SchemathesisFunction: + test_name = f"{previous_tests} -> {endpoint.method}:{endpoint.full_path}" + return create( + self.__class__, + name=f"{self.originalname}[{test_name}]", + parent=self.parent, + callspec=getattr(self, "callspec", None), + callobj=test, + fixtureinfo=self._fixtureinfo, + keywords=self.keywords, + originalname=self.originalname, + test_func=self.test_function, + test_name=test_name, + recursion_level=recursion_level + 1, + ) + + return [ + make_test(endpoint, test, previous_test_name) + for (endpoint, test) in feedback.get_stateful_tests(self.test_function, None, None) + ] + + def add_stateful_tests(self) -> None: + idx = self.session.items.index(self) + 1 + tests = self._get_stateful_tests() + self.session.items[idx:idx] = tests + self.session.testscollected += len(tests) + @hookimpl(hookwrapper=True) # type:ignore # pragma: no mutate def pytest_pycollect_makeitem(collector: nodes.Collector, name: str, obj: Any) -> Generator[None, Any, None]: @@ -135,3 +206,13 @@ def pytest_pyfunc_call(pyfuncitem): # type:ignore outcome.get_result() except InvalidArgument as exc: pytest.fail(exc.args[0]) + + +def pytest_runtest_protocol(item: Function, nextitem: Optional[Function]) -> bool: + item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) + runtestprotocol(item, nextitem=nextitem) + item.ihook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location) + if isinstance(item, SchemathesisFunction): + item.warn_if_stateful_responses_not_stored() + item.add_stateful_tests() + return True diff --git a/src/schemathesis/models.py b/src/schemathesis/models.py index d559e1cf9e..6f857ae80f 100644 --- a/src/schemathesis/models.py +++ b/src/schemathesis/models.py @@ -24,7 +24,13 @@ if TYPE_CHECKING: from .hooks import HookDispatcher from .schemas import BaseSchema - from .stateful import StatefulTest + from .stateful import Feedback, Stateful, StatefulTest + + +MISSING_STATEFUL_ARGUMENT_MESSAGE = ( + "To use `store_response` you need to enable stateful testing by adding " + "`stateful=Stateful.links` to your `parametrize` call." +) @attr.s(slots=True) # pragma: no mutate @@ -39,6 +45,8 @@ class Case: body: Optional[Body] = attr.ib(default=None) # pragma: no mutate form_data: Optional[FormData] = attr.ib(default=None) # pragma: no mutate + feedback: "Feedback" = attr.ib(repr=False, default=None) + @property def path(self) -> str: return self.endpoint.path @@ -177,8 +185,15 @@ def call( response = session.request(**data) # type: ignore if close_session: session.close() + if self.feedback: + self.store_response(response) return response + def store_response(self, response: GenericResponse) -> None: + if self.feedback is None: + raise RuntimeError(MISSING_STATEFUL_ARGUMENT_MESSAGE) + self.feedback.add_test_case(self, response) + def as_werkzeug_kwargs(self, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]: """Convert the case into a dictionary acceptable by werkzeug.Client.""" final_headers = self.headers.copy() if self.headers is not None else {} @@ -212,7 +227,10 @@ def call_wsgi(self, app: Any = None, headers: Optional[Dict[str, str]] = None, * data = self.as_werkzeug_kwargs(headers) client = werkzeug.Client(application, WSGIResponse) with cookie_handler(client, self.cookies): - return client.open(**data, **kwargs) + response = client.open(**data, **kwargs) + if self.feedback: + self.store_response(response) + return response def call_asgi( self, @@ -341,14 +359,16 @@ class Endpoint: def full_path(self) -> str: return self.schema.get_full_path(self.path) - def as_strategy(self, hooks: Optional["HookDispatcher"] = None) -> SearchStrategy: - return self.schema.get_case_strategy(self, hooks) + def as_strategy( + self, hooks: Optional["HookDispatcher"] = None, feedback: Optional["Feedback"] = None + ) -> SearchStrategy: + return self.schema.get_case_strategy(self, hooks, feedback) def get_strategies_from_examples(self) -> List[SearchStrategy[Case]]: """Get examples from endpoint.""" return self.schema.get_strategies_from_examples(self) - def get_stateful_tests(self, response: GenericResponse, stateful: Optional[str]) -> Sequence["StatefulTest"]: + def get_stateful_tests(self, response: GenericResponse, stateful: Optional["Stateful"]) -> Sequence["StatefulTest"]: return self.schema.get_stateful_tests(response, self, stateful) def get_hypothesis_conversions(self, location: str) -> Optional[Callable]: diff --git a/src/schemathesis/runner/__init__.py b/src/schemathesis/runner/__init__.py index 0f11c47bd9..2619ed4d33 100644 --- a/src/schemathesis/runner/__init__.py +++ b/src/schemathesis/runner/__init__.py @@ -7,14 +7,15 @@ from .. import fixups as _fixups from .. import loaders from ..checks import DEFAULT_CHECKS +from ..constants import DEFAULT_STATEFUL_RECURSION_LIMIT from ..models import CheckFunction from ..schemas import BaseSchema +from ..stateful import Stateful from ..targets import DEFAULT_TARGETS, Target from ..types import Filter, NotSet, RawAuth from ..utils import dict_not_none_values, dict_true_values, file_exists, get_requests_auth, import_app from . import events from .impl import ( - DEFAULT_STATEFUL_RECURSION_LIMIT, BaseRunner, SingleThreadASGIRunner, SingleThreadRunner, @@ -36,7 +37,7 @@ def prepare( # pylint: disable=too-many-arguments exit_first: bool = False, store_interactions: bool = False, fixups: Iterable[str] = (), - stateful: Optional[str] = None, + stateful: Optional[Stateful] = None, stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT, # Schema loading loader: Callable = loaders.from_uri, @@ -146,7 +147,7 @@ def execute_from_schema( exit_first: bool = False, store_interactions: bool = False, fixups: Iterable[str] = (), - stateful: Optional[str] = None, + stateful: Optional[Stateful] = None, stateful_recursion_limit: int = DEFAULT_STATEFUL_RECURSION_LIMIT, ) -> Generator[events.ExecutionEvent, None, None]: """Execute tests for the given schema. diff --git a/src/schemathesis/runner/impl/__init__.py b/src/schemathesis/runner/impl/__init__.py index bf25c7932c..de53b7d985 100644 --- a/src/schemathesis/runner/impl/__init__.py +++ b/src/schemathesis/runner/impl/__init__.py @@ -1,3 +1,3 @@ -from .core import DEFAULT_STATEFUL_RECURSION_LIMIT, BaseRunner +from .core import BaseRunner from .solo import SingleThreadASGIRunner, SingleThreadRunner, SingleThreadWSGIRunner from .threadpool import ThreadPoolASGIRunner, ThreadPoolRunner, ThreadPoolWSGIRunner diff --git a/src/schemathesis/runner/impl/core.py b/src/schemathesis/runner/impl/core.py index 0f672df8e5..331896cc55 100644 --- a/src/schemathesis/runner/impl/core.py +++ b/src/schemathesis/runner/impl/core.py @@ -1,7 +1,7 @@ import logging import time from contextlib import contextmanager -from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Union import attr import hypothesis @@ -9,20 +9,17 @@ from _pytest.logging import LogCaptureHandler, catching_logs from requests.auth import HTTPDigestAuth, _basic_auth_str -from ..._hypothesis import make_test_or_exception -from ...constants import DEFAULT_DEADLINE, USER_AGENT +from ...constants import DEFAULT_DEADLINE, DEFAULT_STATEFUL_RECURSION_LIMIT, USER_AGENT from ...exceptions import CheckFailed, InvalidSchema, get_grouped_exception from ...hooks import HookContext, get_all_by_name from ...models import Case, CheckFunction, Endpoint, Status, TestResult, TestResultSet from ...runner import events from ...schemas import BaseSchema -from ...stateful import ParsedData, StatefulTest +from ...stateful import Feedback, Stateful from ...targets import Target, TargetContext from ...types import RawAuth from ...utils import GenericResponse, WSGIResponse, capture_hypothesis_output, format_exception -DEFAULT_STATEFUL_RECURSION_LIMIT = 5 # pragma: no mutate - def get_hypothesis_settings(hypothesis_options: Dict[str, Any]) -> hypothesis.settings: # Default settings, used as a parent settings object below @@ -30,48 +27,6 @@ def get_hypothesis_settings(hypothesis_options: Dict[str, Any]) -> hypothesis.se return hypothesis.settings(**hypothesis_options) -@attr.s(slots=True) # pragma: no mutate -class StatefulData: - """Storage for data that will be used in later tests.""" - - stateful_test: StatefulTest = attr.ib() # pragma: no mutate - container: List[ParsedData] = attr.ib(factory=list) # pragma: no mutate - - def make_endpoint(self) -> Endpoint: - return self.stateful_test.make_endpoint(self.container) - - def store(self, case: Case, response: GenericResponse) -> None: - """Parse and store data for a stateful test.""" - parsed = self.stateful_test.parse(case, response) - self.container.append(parsed) - - -@attr.s(slots=True) # pragma: no mutate -class Feedback: - """Handler for feedback from tests. - - Provides a way to control runner's behavior from tests. - """ - - stateful: Optional[str] = attr.ib() # pragma: no mutate - endpoint: Endpoint = attr.ib() # pragma: no mutate - stateful_tests: Dict[str, StatefulData] = attr.ib(factory=dict) # pragma: no mutate - - def add_test_case(self, case: Case, response: GenericResponse) -> None: - """Store test data to reuse it in the future additional tests.""" - for stateful_test in case.endpoint.get_stateful_tests(response, self.stateful): - data = self.stateful_tests.setdefault(stateful_test.name, StatefulData(stateful_test)) - data.store(case, response) - - def get_stateful_tests( - self, test: Callable, settings: hypothesis.settings, seed: Optional[int] - ) -> Generator[Tuple[Endpoint, Union[Callable, InvalidSchema]], None, None]: - """Generate additional tests that use data from the previous ones.""" - for data in self.stateful_tests.values(): - endpoint = data.make_endpoint() - yield endpoint, make_test_or_exception(endpoint, test, settings, seed) - - # pylint: disable=too-many-instance-attributes @attr.s # pragma: no mutate class BaseRunner: @@ -86,7 +41,7 @@ class BaseRunner: store_interactions: bool = attr.ib(default=False) # pragma: no mutate seed: Optional[int] = attr.ib(default=None) # pragma: no mutate exit_first: bool = attr.ib(default=False) # pragma: no mutate - stateful: Optional[str] = attr.ib(default=None) # pragma: no mutate + stateful: Optional[Stateful] = attr.ib(default=None) # pragma: no mutate stateful_recursion_limit: int = attr.ib(default=DEFAULT_STATEFUL_RECURSION_LIMIT) # pragma: no mutate def execute(self) -> Generator[events.ExecutionEvent, None, None]: diff --git a/src/schemathesis/runner/impl/threadpool.py b/src/schemathesis/runner/impl/threadpool.py index 80291395ee..188c838826 100644 --- a/src/schemathesis/runner/impl/threadpool.py +++ b/src/schemathesis/runner/impl/threadpool.py @@ -9,11 +9,12 @@ from ..._hypothesis import make_test_or_exception from ...models import CheckFunction, TestResultSet +from ...stateful import Feedback, Stateful from ...targets import Target from ...types import RawAuth from ...utils import capture_hypothesis_output, get_requests_auth from .. import events -from .core import BaseRunner, Feedback, asgi_test, get_session, network_test, run_test, wsgi_test +from .core import BaseRunner, asgi_test, get_session, network_test, run_test, wsgi_test def _run_task( @@ -25,7 +26,7 @@ def _run_task( settings: hypothesis.settings, seed: Optional[int], results: TestResultSet, - stateful: Optional[str], + stateful: Optional[Stateful], stateful_recursion_limit: int, **kwargs: Any, ) -> None: @@ -61,7 +62,7 @@ def thread_task( headers: Optional[Dict[str, Any]], seed: Optional[int], results: TestResultSet, - stateful: Optional[str], + stateful: Optional[Stateful], stateful_recursion_limit: int, kwargs: Any, ) -> None: @@ -97,7 +98,7 @@ def wsgi_thread_task( settings: hypothesis.settings, seed: Optional[int], results: TestResultSet, - stateful: Optional[str], + stateful: Optional[Stateful], stateful_recursion_limit: int, kwargs: Any, ) -> None: @@ -126,7 +127,7 @@ def asgi_thread_task( headers: Optional[Dict[str, Any]], seed: Optional[int], results: TestResultSet, - stateful: Optional[str], + stateful: Optional[Stateful], stateful_recursion_limit: int, kwargs: Any, ) -> None: diff --git a/src/schemathesis/schemas.py b/src/schemathesis/schemas.py index 82318d4d8f..0e8476e0cd 100644 --- a/src/schemathesis/schemas.py +++ b/src/schemathesis/schemas.py @@ -17,10 +17,11 @@ from requests.structures import CaseInsensitiveDict from ._hypothesis import make_test_or_exception +from .constants import DEFAULT_STATEFUL_RECURSION_LIMIT from .exceptions import InvalidSchema from .hooks import HookContext, HookDispatcher, HookScope, dispatch from .models import Case, Endpoint -from .stateful import StatefulTest +from .stateful import Feedback, Stateful, StatefulTest from .types import Filter, FormData, GenericTest, NotSet from .utils import NOT_SET, GenericResponse @@ -38,6 +39,8 @@ class BaseSchema(Mapping): hooks: HookDispatcher = attr.ib(factory=lambda: HookDispatcher(scope=HookScope.SCHEMA)) # pragma: no mutate test_function: Optional[GenericTest] = attr.ib(default=None) # pragma: no mutate validate_schema: bool = attr.ib(default=True) # pragma: no mutate + stateful: Optional[Stateful] = attr.ib(default=None) # pragma: no mutate + stateful_recursion_limit: int = attr.ib(default=DEFAULT_STATEFUL_RECURSION_LIMIT) # pragma: no mutate def __iter__(self) -> Iterator[str]: return iter(self.endpoints) @@ -99,7 +102,7 @@ def get_strategies_from_examples(self, endpoint: Endpoint) -> List[SearchStrateg raise NotImplementedError def get_stateful_tests( - self, response: GenericResponse, endpoint: Endpoint, stateful: Optional[str] + self, response: GenericResponse, endpoint: Endpoint, stateful: Optional[Stateful] ) -> Sequence[StatefulTest]: """Get a list of additional tests, that should be executed after this response from the endpoint.""" raise NotImplementedError @@ -123,6 +126,8 @@ def parametrize( # pylint: disable=too-many-arguments tag: Optional[Filter] = NOT_SET, operation_id: Optional[Filter] = NOT_SET, validate_schema: Union[bool, NotSet] = NOT_SET, + stateful: Optional[Union[Stateful, NotSet]] = NOT_SET, + stateful_recursion_limit: Union[int, NotSet] = NOT_SET, ) -> Callable: """Mark a test function as a parametrized one.""" @@ -135,6 +140,8 @@ def wrapper(func: GenericTest) -> GenericTest: tag=tag, operation_id=operation_id, validate_schema=validate_schema, + stateful=stateful, + stateful_recursion_limit=stateful_recursion_limit, ) return func @@ -150,6 +157,8 @@ def clone( # pylint: disable=too-many-arguments operation_id: Optional[Filter] = NOT_SET, hooks: Union[HookDispatcher, NotSet] = NOT_SET, validate_schema: Union[bool, NotSet] = NOT_SET, + stateful: Optional[Union[Stateful, NotSet]] = NOT_SET, + stateful_recursion_limit: Union[int, NotSet] = NOT_SET, ) -> "BaseSchema": if method is NOT_SET: method = self.method @@ -163,6 +172,10 @@ def clone( # pylint: disable=too-many-arguments validate_schema = self.validate_schema if hooks is NOT_SET: hooks = self.hooks + if stateful is NOT_SET: + stateful = self.stateful + if stateful_recursion_limit is NOT_SET: + stateful_recursion_limit = self.stateful_recursion_limit return self.__class__( self.raw_schema, @@ -176,6 +189,8 @@ def clone( # pylint: disable=too-many-arguments hooks=hooks, # type: ignore test_function=test_function, validate_schema=validate_schema, # type: ignore + stateful=stateful, # type: ignore + stateful_recursion_limit=stateful_recursion_limit, # type: ignore ) def get_local_hook_dispatcher(self) -> Optional[HookDispatcher]: @@ -206,5 +221,7 @@ def prepare_multipart( def get_request_payload_content_types(self, endpoint: Endpoint) -> List[str]: raise NotImplementedError - def get_case_strategy(self, endpoint: Endpoint, hooks: Optional[HookDispatcher] = None) -> SearchStrategy: + def get_case_strategy( + self, endpoint: Endpoint, hooks: Optional[HookDispatcher] = None, feedback: Optional[Feedback] = None + ) -> SearchStrategy: raise NotImplementedError diff --git a/src/schemathesis/specs/openapi/_hypothesis.py b/src/schemathesis/specs/openapi/_hypothesis.py index 1f8e4162a5..2eb8ae66cb 100644 --- a/src/schemathesis/specs/openapi/_hypothesis.py +++ b/src/schemathesis/specs/openapi/_hypothesis.py @@ -12,6 +12,7 @@ from ...exceptions import InvalidSchema from ...hooks import GLOBAL_HOOK_DISPATCHER, HookContext, HookDispatcher from ...models import Case, Endpoint +from ...stateful import Feedback PARAMETERS = frozenset(("path_parameters", "headers", "cookies", "query", "body", "form_data")) SLASH = "/" @@ -69,13 +70,15 @@ def is_valid_query(query: Dict[str, Any]) -> bool: return True -def get_case_strategy(endpoint: Endpoint, hooks: Optional[HookDispatcher] = None) -> st.SearchStrategy: +def get_case_strategy( + endpoint: Endpoint, hooks: Optional[HookDispatcher] = None, feedback: Optional[Feedback] = None +) -> st.SearchStrategy: """Create a strategy for a complete test case. Path & endpoint are static, the others are JSON schemas. """ strategies = {} - static_kwargs: Dict[str, Any] = {} + static_kwargs: Dict[str, Any] = {"feedback": feedback} for parameter in PARAMETERS: value = getattr(endpoint, parameter) if value is not None: diff --git a/src/schemathesis/specs/openapi/schemas.py b/src/schemathesis/specs/openapi/schemas.py index 6d96824d32..8aba53e060 100644 --- a/src/schemathesis/specs/openapi/schemas.py +++ b/src/schemathesis/specs/openapi/schemas.py @@ -12,7 +12,7 @@ from ...hooks import HookContext, HookDispatcher from ...models import Case, Endpoint, EndpointDefinition, empty_object from ...schemas import BaseSchema -from ...stateful import StatefulTest +from ...stateful import Feedback, Stateful, StatefulTest from ...types import FormData from ...utils import GenericResponse from . import links, serialization @@ -36,9 +36,9 @@ def spec_version(self) -> str: raise NotImplementedError def get_stateful_tests( - self, response: GenericResponse, endpoint: Endpoint, stateful: Optional[str] + self, response: GenericResponse, endpoint: Endpoint, stateful: Optional[Stateful] ) -> Sequence[StatefulTest]: - if stateful == "links": + if stateful == Stateful.links: return links.get_links(response, endpoint, field=self.links_field) return [] @@ -178,8 +178,10 @@ def get_endpoint_by_reference(self, reference: str) -> Endpoint: raw_definition = EndpointDefinition(data, resolved_definition, scope) return self.make_endpoint(path, method, parameters, resolved_definition, raw_definition) - def get_case_strategy(self, endpoint: Endpoint, hooks: Optional[HookDispatcher] = None) -> SearchStrategy: - return get_case_strategy(endpoint, hooks) + def get_case_strategy( + self, endpoint: Endpoint, hooks: Optional[HookDispatcher] = None, feedback: Optional[Feedback] = None + ) -> SearchStrategy: + return get_case_strategy(endpoint, hooks, feedback) class SwaggerV20(BaseOpenAPISchema): diff --git a/src/schemathesis/stateful.py b/src/schemathesis/stateful.py index 8bcb679222..dcfdb868ca 100644 --- a/src/schemathesis/stateful.py +++ b/src/schemathesis/stateful.py @@ -1,12 +1,19 @@ +import enum import json -from typing import Any, Dict, List +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union import attr +import hypothesis +from .exceptions import InvalidSchema from .models import Case, Endpoint from .utils import NOT_SET, GenericResponse +class Stateful(enum.Enum): + links = 1 + + @attr.s(slots=True, hash=False) # pragma: no mutate class ParsedData: """A structure that holds information parsed from a test outcomes. @@ -41,3 +48,47 @@ def parse(self, case: Case, response: GenericResponse) -> ParsedData: def make_endpoint(self, data: List[ParsedData]) -> Endpoint: raise NotImplementedError + + +@attr.s(slots=True) # pragma: no mutate +class StatefulData: + """Storage for data that will be used in later tests.""" + + stateful_test: StatefulTest = attr.ib() # pragma: no mutate + container: List[ParsedData] = attr.ib(factory=list) # pragma: no mutate + + def make_endpoint(self) -> Endpoint: + return self.stateful_test.make_endpoint(self.container) + + def store(self, case: Case, response: GenericResponse) -> None: + """Parse and store data for a stateful test.""" + parsed = self.stateful_test.parse(case, response) + self.container.append(parsed) + + +@attr.s(slots=True) # pragma: no mutate +class Feedback: + """Handler for feedback from tests. + + Provides a way to control runner's behavior from tests. + """ + + stateful: Optional[Stateful] = attr.ib() # pragma: no mutate + endpoint: Endpoint = attr.ib() # pragma: no mutate + stateful_tests: Dict[str, StatefulData] = attr.ib(factory=dict) # pragma: no mutate + + def add_test_case(self, case: Case, response: GenericResponse) -> None: + """Store test data to reuse it in the future additional tests.""" + for stateful_test in case.endpoint.get_stateful_tests(response, self.stateful): + data = self.stateful_tests.setdefault(stateful_test.name, StatefulData(stateful_test)) + data.store(case, response) + + def get_stateful_tests( + self, test: Callable, settings: Optional[hypothesis.settings], seed: Optional[int] + ) -> Generator[Tuple[Endpoint, Union[Callable, InvalidSchema]], None, None]: + """Generate additional tests that use data from the previous ones.""" + from ._hypothesis import make_test_or_exception # pylint: disable=import-outside-toplevel + + for data in self.stateful_tests.values(): + endpoint = data.make_endpoint() + yield endpoint, make_test_or_exception(endpoint, test, settings, seed) diff --git a/test/conftest.py b/test/conftest.py index f6a39e279e..90a83f77be 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -350,8 +350,8 @@ def openapi_30(): @pytest.fixture() -def app_schema(openapi_version): - return _aiohttp.make_openapi_schema(endpoints=("success", "failure"), version=openapi_version) +def app_schema(openapi_version, endpoints): + return _aiohttp.make_openapi_schema(endpoints=endpoints, version=openapi_version) @pytest.fixture() @@ -372,6 +372,7 @@ def maker( """ import pytest import schemathesis + from schemathesis import Stateful from test.utils import * from hypothesis import given, settings, HealthCheck, Phase raw_schema = {schema} diff --git a/test/specs/openapi/test_links.py b/test/specs/openapi/test_links.py index 7e224fb505..924ae20826 100644 --- a/test/specs/openapi/test_links.py +++ b/test/specs/openapi/test_links.py @@ -7,7 +7,7 @@ import schemathesis from schemathesis.models import Case, Endpoint from schemathesis.specs.openapi.links import Link -from schemathesis.stateful import ParsedData +from schemathesis.stateful import ParsedData, Stateful ENDPOINT = Endpoint( path="/users/{user_id}", @@ -87,7 +87,7 @@ def response(): def test_get_links(openapi3_base_url, schema_url, url, expected): schema = schemathesis.from_uri(schema_url) response = requests.post(f"{openapi3_base_url}{url}", json={"username": "TEST"}) - assert schema.endpoints["/users/"]["POST"].get_stateful_tests(response, "links") == expected + assert schema.endpoints["/users/"]["POST"].get_stateful_tests(response, Stateful.links) == expected def test_parse(case, response): diff --git a/test/test_stateful.py b/test/test_stateful.py index c624a8ec5f..2789f20a6a 100644 --- a/test/test_stateful.py +++ b/test/test_stateful.py @@ -1,8 +1,12 @@ import pytest +from schemathesis.extra.pytest_plugin import NOT_USED_STATEFUL_TESTING_MESSAGE +from schemathesis.models import MISSING_STATEFUL_ARGUMENT_MESSAGE from schemathesis.stateful import ParsedData from schemathesis.utils import NOT_SET +from .apps.utils import OpenAPIVersion + @pytest.mark.parametrize( "parameters, body", (({"a": 1}, None), ({"a": 1}, NOT_SET), ({"a": 1}, {"value": 1}), ({"a": 1}, [1, 2, 3])) @@ -10,3 +14,141 @@ def test_hashable(parameters, body): # All parsed data should be hashable hash(ParsedData(parameters, body)) + + +@pytest.fixture +def openapi_version(): + return OpenAPIVersion("3.0") + + +@pytest.mark.endpoints("create_user", "get_user", "update_user") +def test_stateful_enabled(testdir, app_schema, openapi3_base_url): + # When "stateful" is used in the "parametrize" decorator + testdir.make_test( + f""" +@schema.parametrize(stateful=Stateful.links, method="POST") +@settings(max_examples=2) +def test_(request, case): + request.config.HYPOTHESIS_CASES += 1 + response = case.call(base_url="{openapi3_base_url}") + case.validate_response(response) + """, + schema=app_schema, + ) + # Then there should be 4 tests in total: + # 1 - the original one for POST /users + # 3 - stateful ones: + # - POST /users -> GET /users + # - POST /users -> GET /users -> PATCH /users + # - POST /users -> PATCH /users + result = testdir.run_and_assert("-v", passed=4) + result.stdout.re_match_lines( + [ + r"test_stateful_enabled.py::test_\[POST:/api/users/\] PASSED", + r"test_stateful_enabled.py::test_\[POST:/api/users/ -> GET:/api/users/{user_id}\] PASSED * \[ 66%\]", + r"test_stateful_enabled.py::test_" + r"\[POST:/api/users/ -> GET:/api/users/{user_id} -> PATCH:/api/users/{user_id}\] PASSED * \[ 75%\]", + r"test_stateful_enabled.py::test_\[POST:/api/users/ -> PATCH:/api/users/{user_id}\] PASSED * \[100%\]", + r"Hypothesis calls: 8", + ] + ) + + +@pytest.mark.endpoints("create_user", "get_user", "update_user") +def test_stateful_enabled_limit(testdir, app_schema, openapi3_base_url): + # When "stateful" is used in the "parametrize" decorator + # And "stateful_recursion_limit" is set to some number + testdir.make_test( + f""" +@schema.parametrize(stateful=Stateful.links, stateful_recursion_limit=1, method="POST") +@settings(max_examples=2) +def test_(request, case): + request.config.HYPOTHESIS_CASES += 1 + response = case.call(base_url="{openapi3_base_url}") + case.validate_response(response) + """, + schema=app_schema, + ) + # Then there should be 3 tests in total: + # 1 - the original one for POST /users + # 2 - stateful ones: + # - POST /users -> GET /users + # - POST /users -> PATCH /users + result = testdir.run_and_assert("-v", passed=3) + result.stdout.re_match_lines( + [ + r"test_stateful_enabled_limit.py::test_\[POST:/api/users/\] PASSED", + r"test_stateful_enabled_limit.py::test_\[POST:/api/users/ -> GET:/api/users/{user_id}\] PASSED * \[ 66%\]", + r"test_stateful_enabled_limit.py::test_\[POST:/api/users/ -> PATCH:/api/users/{user_id}\] PASSED * \[100%\]", + r"Hypothesis calls: 6", + ] + ) + + +@pytest.mark.endpoints("create_user", "get_user", "update_user") +def test_stateful_disabled(testdir, app_schema, openapi3_base_url): + # When "stateful" is NOT used in the "parametrize" decorator + testdir.make_test( + f""" +@schema.parametrize(method="POST") +@settings(max_examples=2) +def test_(request, case): + request.config.HYPOTHESIS_CASES += 1 + response = case.call(base_url="{openapi3_base_url}") + case.validate_response(response) + """, + schema=app_schema, + ) + # Then there should be 1 test in total - the original one for POST /users + result = testdir.run_and_assert("-v", passed=1) + result.stdout.re_match_lines( + [ + r"test_stateful_disabled.py::test_\[POST:/api/users/\] PASSED", + r"Hypothesis calls: 2", + ] + ) + + +@pytest.mark.endpoints("create_user", "get_user", "update_user") +def test_stateful_not_used(testdir, app_schema): + # When "stateful" is used in the "parametrize" decorator + # And the test doesn't use "Case.call" or "Case.store_response" to store the feedback + testdir.make_test( + """ +@schema.parametrize(stateful=Stateful.links, method="POST") +@settings(max_examples=2) +def test_(request, case): + request.config.HYPOTHESIS_CASES += 1 + """, + schema=app_schema, + ) + # Then there should be 1 test in total - the original one for POST /users + result = testdir.run_and_assert("-v", passed=1) + result.stdout.re_match_lines( + [ + r"test_stateful_not_used.py::test_\[POST:/api/users/\] PASSED", + r"Hypothesis calls: 2", + ] + ) + # And a warning should be risen because responses were not stored + assert f" test_stateful_not_used.py:10: PytestWarning: {NOT_USED_STATEFUL_TESTING_MESSAGE}" in result.stdout.lines + + +@pytest.mark.endpoints("create_user", "get_user", "update_user") +def test_store_response_without_stateful(testdir, app_schema, openapi3_base_url): + # When "stateful" is NOT used in the "parametrize" decorator + # And "case.store_response" is used inside the test + testdir.make_test( + f""" +@schema.parametrize(method="POST") +@settings(max_examples=2) +def test_(request, case): + request.config.HYPOTHESIS_CASES += 1 + response = case.call(base_url="{openapi3_base_url}") + with pytest.raises(RuntimeError, match="{MISSING_STATEFUL_ARGUMENT_MESSAGE}"): + case.store_response(response) + """, + schema=app_schema, + ) + # Then there should be an exception, that is verified via the pytest.raises ctx manager + testdir.run_and_assert(passed=1)