diff --git a/docs/changelog.rst b/docs/changelog.rst index feb288fd17..932f06f703 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Added - Storing network logs with ``--store-network-log=``. The stored cassettes are based on the `VCR format `_ and contain extra information from the Schemathesis internals. `#379`_ +- Targeted property-based testing in CLI and runner. It only supports `response_time` target at the moment. `#104`_ Fixed ~~~~~ @@ -1055,6 +1056,7 @@ Fixed .. _#109: https://github.com/kiwicom/schemathesis/issues/109 .. _#107: https://github.com/kiwicom/schemathesis/issues/107 .. _#106: https://github.com/kiwicom/schemathesis/issues/106 +.. _#104: https://github.com/kiwicom/schemathesis/issues/104 .. _#101: https://github.com/kiwicom/schemathesis/issues/101 .. _#99: https://github.com/kiwicom/schemathesis/issues/99 .. _#98: https://github.com/kiwicom/schemathesis/issues/98 diff --git a/docs/index.rst b/docs/index.rst index 2422acc837..5d18642240 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,7 @@ Welcome to schemathesis's documentation! :caption: Contents: usage + targeted faq changelog diff --git a/docs/targeted.rst b/docs/targeted.rst new file mode 100644 index 0000000000..1d082c467c --- /dev/null +++ b/docs/targeted.rst @@ -0,0 +1,76 @@ +.. _targeted: + +Targeted property-based testing +=============================== + +Schemathesis supports targeted property-based testing via utilizing ``hypothesis.target`` inside its runner and provides +an API to guide data generation towards certain pre-defined goals: + +- ``response_time``. Hypothesis will try to generate input that will more likely to have higher response time; + +To illustrate this feature, consider the following AioHTTP endpoint, that contains a hidden performance problem - +the more zeroes are in the input number the slower it works and if there are more than 10 zeroes it will cause an internal +server error: + +.. code:: python + + async def performance(request: web.Request) -> web.Response: + decoded = await request.json() + number = str(decoded).count("0") + if number > 0: + # emulate hard work + await asyncio.sleep(0.01 * number) + if number > 10: + raise web.HTTPInternalServerError + return web.json_response({"slow": True}) + +Let's take a look if Schemathesis can discover this issue and how much time it will take: + +.. code:: bash + + $ schemathesis run --hypothesis-max-examples=100000 http://127.0.0.1:8081/swagger.yaml + ... + 1. Received a response with 5xx status code: 500 + + Check : not_a_server_error + Body : 58150920460703009030426716484679203200 + + Run this Python code to reproduce this failure: + + requests.post('http://127.0.0.1:8081/api/performance', json=58150920460703009030426716484679203200) + + Or add this option to your command line parameters: --hypothesis-seed=240368931405400688094965957483327791742 + ================================================== SUMMARY ================================================== + + Performed checks: + not_a_server_error 67993 / 68041 passed FAILED + + ============================================ 1 failed in 662.16s =========================================== + +And with targeted testing (``.hypothesis`` directory was removed between these test runs to avoid reusing results): + +.. code:: bash + + $ schemathesis run --target=response_time --hypothesis-max-examples=100000 http://127.0.0.1:8081/swagger.yaml + ... + 1. Received a response with 5xx status code: 500 + + Check : not_a_server_error + Body : 2600050604444474172950385505254500000 + + Run this Python code to reproduce this failure: + + requests.post('http://127.0.0.1:8081/api/performance', json=2600050604444474172950385505254500000) + + Or add this option to your command line parameters: --hypothesis-seed=340229547842147149729957578683815058325 + ================================================== SUMMARY ================================================== + + Performed checks: + not_a_server_error 22039 / 22254 passed FAILED + + ============================================ 1 failed in 305.50s =========================================== + +This behavior is reproducible in general, but not guaranteed due to the randomness of data generation. However, it shows +a significant testing time reduction especially on a big number of examples. + +Hypothesis `documentation `_ provides a detailed explanation of what targeted property-based testing is. diff --git a/src/schemathesis/cli/__init__.py b/src/schemathesis/cli/__init__.py index ada6a124aa..4ca052d9e6 100644 --- a/src/schemathesis/cli/__init__.py +++ b/src/schemathesis/cli/__init__.py @@ -9,6 +9,7 @@ from .. import checks as checks_module from .. import models, runner from ..runner import events +from ..runner.targeted import DEFAULT_TARGETS_NAMES, Target from ..types import Filter from ..utils import WSGIResponse from . import callbacks, cassettes, output @@ -45,6 +46,15 @@ def schemathesis(pre_run: Optional[str] = None) -> None: @click.option( "--checks", "-c", multiple=True, help="List of checks to run.", type=CHECKS_TYPE, default=DEFAULT_CHECKS_NAMES ) +@click.option( + "--target", + "-t", + "targets", + multiple=True, + help="Targets for input generation.", + type=click.Choice([target.name for target in Target]), + default=DEFAULT_TARGETS_NAMES, +) @click.option( "-x", "--exitfirst", "exit_first", is_flag=True, default=False, help="Exit instantly on first error or failed test." ) @@ -152,6 +162,7 @@ def run( # pylint: disable=too-many-arguments auth_type: str, headers: Dict[str, str], checks: Iterable[str] = DEFAULT_CHECKS_NAMES, + targets: Iterable[str] = DEFAULT_TARGETS_NAMES, exit_first: bool = False, endpoints: Optional[Filter] = None, methods: Optional[Filter] = None, @@ -177,6 +188,7 @@ def run( # pylint: disable=too-many-arguments SCHEMA must be a valid URL or file path pointing to an Open API / Swagger specification. """ # pylint: disable=too-many-locals + selected_targets = tuple(target for target in Target if target.name in targets) if "all" in checks: selected_checks = checks_module.ALL_CHECKS @@ -198,6 +210,7 @@ def run( # pylint: disable=too-many-arguments exit_first=exit_first, store_interactions=store_network_log is not None, checks=selected_checks, + targets=selected_targets, workers_num=workers_num, validate_schema=validate_schema, hypothesis_deadline=hypothesis_deadline, diff --git a/src/schemathesis/runner/__init__.py b/src/schemathesis/runner/__init__.py index 4308c48408..7708d70dc3 100644 --- a/src/schemathesis/runner/__init__.py +++ b/src/schemathesis/runner/__init__.py @@ -11,6 +11,7 @@ from ..utils import dict_not_none_values, dict_true_values, file_exists, get_base_url, get_requests_auth, import_app from . import events from .impl import BaseRunner, SingleThreadRunner, SingleThreadWSGIRunner, ThreadPoolRunner, ThreadPoolWSGIRunner +from .targeted import DEFAULT_TARGETS, Target def prepare( # pylint: disable=too-many-arguments @@ -18,6 +19,7 @@ def prepare( # pylint: disable=too-many-arguments *, # Runtime behavior checks: Iterable[CheckFunction] = DEFAULT_CHECKS, + targets: Iterable[Target] = DEFAULT_TARGETS, workers_num: int = 1, seed: Optional[int] = None, exit_first: bool = False, @@ -70,6 +72,7 @@ def prepare( # pylint: disable=too-many-arguments app=app, validate_schema=validate_schema, checks=checks, + targets=targets, hypothesis_options=hypothesis_options, seed=seed, workers_num=workers_num, @@ -112,6 +115,7 @@ def execute_from_schema( app: Optional[str] = None, validate_schema: bool = True, checks: Iterable[CheckFunction], + targets: Iterable[Target], workers_num: int = 1, hypothesis_options: Dict[str, Any], auth: Optional[RawAuth] = None, @@ -150,6 +154,7 @@ def execute_from_schema( runner = ThreadPoolWSGIRunner( schema=schema, checks=checks, + targets=targets, hypothesis_settings=hypothesis_options, auth=auth, auth_type=auth_type, @@ -163,6 +168,7 @@ def execute_from_schema( runner = ThreadPoolRunner( schema=schema, checks=checks, + targets=targets, hypothesis_settings=hypothesis_options, auth=auth, auth_type=auth_type, @@ -177,6 +183,7 @@ def execute_from_schema( runner = SingleThreadWSGIRunner( schema=schema, checks=checks, + targets=targets, hypothesis_settings=hypothesis_options, auth=auth, auth_type=auth_type, @@ -189,6 +196,7 @@ def execute_from_schema( runner = SingleThreadRunner( schema=schema, checks=checks, + targets=targets, hypothesis_settings=hypothesis_options, auth=auth, auth_type=auth_type, diff --git a/src/schemathesis/runner/impl/core.py b/src/schemathesis/runner/impl/core.py index d30c0bde82..7d7969a9da 100644 --- a/src/schemathesis/runner/impl/core.py +++ b/src/schemathesis/runner/impl/core.py @@ -16,6 +16,7 @@ from ...schemas import BaseSchema from ...types import RawAuth from ...utils import GenericResponse, capture_hypothesis_output +from ..targeted import Target DEFAULT_DEADLINE = 500 # pragma: no mutate @@ -31,6 +32,7 @@ def get_hypothesis_settings(hypothesis_options: Dict[str, Any]) -> hypothesis.se class BaseRunner: schema: BaseSchema = attr.ib() # pragma: no mutate checks: Iterable[CheckFunction] = attr.ib() # pragma: no mutate + targets: Iterable[Target] = attr.ib() # pragma: no mutate hypothesis_settings: hypothesis.settings = attr.ib(converter=get_hypothesis_settings) # pragma: no mutate auth: Optional[RawAuth] = attr.ib(default=None) # pragma: no mutate auth_type: Optional[str] = attr.ib(default=None) # pragma: no mutate @@ -66,6 +68,7 @@ def run_test( endpoint: Endpoint, test: Union[Callable, InvalidSchema], checks: Iterable[CheckFunction], + targets: Iterable[Target], results: TestResultSet, **kwargs: Any, ) -> Generator[events.ExecutionEvent, None, None]: @@ -80,7 +83,7 @@ def run_test( result.add_error(test) else: with capture_hypothesis_output() as hypothesis_output: - test(checks, result, **kwargs) + test(checks, targets, result, **kwargs) status = Status.success except (AssertionError, hypothesis.errors.MultipleFailures): status = Status.failure @@ -133,9 +136,16 @@ def run_checks(case: Case, checks: Iterable[CheckFunction], result: TestResult, raise get_grouped_exception(*errors) +def run_targets(targets: Iterable[Target], elapsed: float) -> None: + for target in targets: + if target == Target.response_time: + hypothesis.target(elapsed, label="response_time") + + def network_test( case: Case, checks: Iterable[CheckFunction], + targets: Iterable[Target], result: TestResult, session: requests.Session, request_timeout: Optional[int], @@ -145,6 +155,7 @@ def network_test( # pylint: disable=too-many-arguments timeout = prepare_timeout(request_timeout) response = case.call(session=session, timeout=timeout) + run_targets(targets, response.elapsed.total_seconds()) if store_interactions: result.store_requests_response(response) run_checks(case, checks, result, response) @@ -174,6 +185,7 @@ def prepare_timeout(timeout: Optional[int]) -> Optional[float]: def wsgi_test( case: Case, checks: Iterable[CheckFunction], + targets: Iterable[Target], result: TestResult, auth: Optional[RawAuth], auth_type: Optional[str], @@ -186,6 +198,7 @@ def wsgi_test( start = time.monotonic() response = case.call_wsgi(headers=headers) elapsed = time.monotonic() - start + run_targets(targets, elapsed) if store_interactions: result.store_wsgi_response(case, response, headers, elapsed) result.logs.extend(recorded.records) diff --git a/src/schemathesis/runner/impl/solo.py b/src/schemathesis/runner/impl/solo.py index 9d8f80ac34..5b945747f5 100644 --- a/src/schemathesis/runner/impl/solo.py +++ b/src/schemathesis/runner/impl/solo.py @@ -21,6 +21,7 @@ def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, N endpoint, test, self.checks, + self.targets, results, session=session, request_timeout=self.request_timeout, @@ -39,6 +40,7 @@ def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, N endpoint, test, self.checks, + self.targets, results, auth=self.auth, auth_type=self.auth_type, diff --git a/src/schemathesis/runner/impl/threadpool.py b/src/schemathesis/runner/impl/threadpool.py index 6cf8000c3f..38de90e094 100644 --- a/src/schemathesis/runner/impl/threadpool.py +++ b/src/schemathesis/runner/impl/threadpool.py @@ -12,6 +12,7 @@ from ...types import RawAuth from ...utils import capture_hypothesis_output, get_requests_auth from .. import events +from ..targeted import Target from .core import BaseRunner, get_session, network_test, run_test, wsgi_test @@ -20,6 +21,7 @@ def _run_task( tasks_queue: Queue, events_queue: Queue, checks: Iterable[CheckFunction], + targets: Iterable[Target], settings: hypothesis.settings, seed: Optional[int], results: TestResultSet, @@ -30,7 +32,7 @@ def _run_task( while not tasks_queue.empty(): endpoint = tasks_queue.get() test = make_test_or_exception(endpoint, test_template, settings, seed) - for event in run_test(endpoint, test, checks, results, **kwargs): + for event in run_test(endpoint, test, checks, targets, results, **kwargs): events_queue.put(event) @@ -38,6 +40,7 @@ def thread_task( tasks_queue: Queue, events_queue: Queue, checks: Iterable[CheckFunction], + targets: Iterable[Target], settings: hypothesis.settings, auth: Optional[RawAuth], auth_type: Optional[str], @@ -53,20 +56,23 @@ def thread_task( # pylint: disable=too-many-arguments prepared_auth = get_requests_auth(auth, auth_type) with get_session(prepared_auth, headers) as session: - _run_task(network_test, tasks_queue, events_queue, checks, settings, seed, results, session=session, **kwargs) + _run_task( + network_test, tasks_queue, events_queue, checks, targets, settings, seed, results, session=session, **kwargs + ) def wsgi_thread_task( tasks_queue: Queue, events_queue: Queue, checks: Iterable[CheckFunction], + targets: Iterable[Target], settings: hypothesis.settings, seed: Optional[int], results: TestResultSet, kwargs: Any, ) -> None: # pylint: disable=too-many-arguments - _run_task(wsgi_test, tasks_queue, events_queue, checks, settings, seed, results, **kwargs) + _run_task(wsgi_test, tasks_queue, events_queue, checks, targets, settings, seed, results, **kwargs) def stop_worker(thread_id: int) -> None: @@ -142,6 +148,7 @@ def _get_worker_kwargs(self, tasks_queue: Queue, events_queue: Queue, results: T "tasks_queue": tasks_queue, "events_queue": events_queue, "checks": self.checks, + "targets": self.targets, "settings": self.hypothesis_settings, "auth": self.auth, "auth_type": self.auth_type, @@ -161,6 +168,7 @@ def _get_worker_kwargs(self, tasks_queue: Queue, events_queue: Queue, results: T "tasks_queue": tasks_queue, "events_queue": events_queue, "checks": self.checks, + "targets": self.targets, "settings": self.hypothesis_settings, "seed": self.seed, "results": results, diff --git a/src/schemathesis/runner/targeted.py b/src/schemathesis/runner/targeted.py new file mode 100644 index 0000000000..b803393e50 --- /dev/null +++ b/src/schemathesis/runner/targeted.py @@ -0,0 +1,9 @@ +from enum import Enum, unique + +DEFAULT_TARGETS = () +DEFAULT_TARGETS_NAMES = () + + +@unique +class Target(Enum): + response_time = 1 diff --git a/test/apps/_aiohttp/handlers.py b/test/apps/_aiohttp/handlers.py index 952025d1b3..70bad98499 100644 --- a/test/apps/_aiohttp/handlers.py +++ b/test/apps/_aiohttp/handlers.py @@ -44,6 +44,18 @@ async def slow(request: web.Request) -> web.Response: return web.json_response({"slow": True}) +async def performance(request: web.Request) -> web.Response: + # Emulate bad performance on certain input type + # This endpoint is for Schemathesis targeted testing, the failure should be discovered + decoded = await request.json() + number = str(decoded).count("0") + if number > 0: + await asyncio.sleep(0.01 * number) + if number > 10: + raise web.HTTPInternalServerError + return web.json_response({"slow": True}) + + async def unsatisfiable(request: web.Request) -> web.Response: return web.json_response({"result": "IMPOSSIBLE!"}) diff --git a/test/apps/utils.py b/test/apps/utils.py index e60841560b..978a7fd604 100644 --- a/test/apps/utils.py +++ b/test/apps/utils.py @@ -10,6 +10,7 @@ class Endpoint(Enum): slow = ("GET", "/api/slow") path_variable = ("GET", "/api/path_variable/{key}") unsatisfiable = ("POST", "/api/unsatisfiable") + performance = ("POST", "/api/performance") invalid = ("POST", "/api/invalid") flaky = ("GET", "/api/flaky") recursive = ("GET", "/api/recursive") @@ -94,6 +95,11 @@ def make_schema(endpoints: Tuple[str, ...]) -> Dict: ], "responses": {"200": {"description": "OK"}}, } + elif endpoint == "performance": + schema = { + "parameters": [{"name": "data", "in": "body", "required": True, "schema": {"type": "integer"},}], + "responses": {"200": {"description": "OK"}}, + } elif endpoint in ("flaky", "multiple_failures"): schema = { "parameters": [{"name": "id", "in": "query", "required": True, "type": "integer"}], diff --git a/test/cli/test_commands.py b/test/cli/test_commands.py index 426f4491e3..329757f1cb 100644 --- a/test/cli/test_commands.py +++ b/test/cli/test_commands.py @@ -5,6 +5,7 @@ from test.utils import HERE, SIMPLE_PATH from urllib.parse import urljoin +import hypothesis import pytest import requests import yaml @@ -16,7 +17,7 @@ from schemathesis.checks import ALL_CHECKS from schemathesis.loaders import from_uri from schemathesis.models import Endpoint -from schemathesis.runner import DEFAULT_CHECKS +from schemathesis.runner import DEFAULT_CHECKS, DEFAULT_TARGETS PHASES = "explicit, reuse, generate, target, shrink" if metadata.version("hypothesis") < "4.5": @@ -155,6 +156,7 @@ def test_commands_run_help(cli): " -c, --checks [not_a_server_error|status_code_conformance|" "content_type_conformance|response_schema_conformance|all]", " List of checks to run.", + " -t, --target [response_time] Targets for input generation.", " -x, --exitfirst Exit instantly on first error or failed test.", " -a, --auth TEXT Server user and password. Example:", " USER:PASSWORD", @@ -259,6 +261,7 @@ def test_execute_arguments(cli, mocker, simple_schema, args, expected): "app": None, "base_url": None, "checks": DEFAULT_CHECKS, + "targets": DEFAULT_TARGETS, "endpoint": (), "method": (), "tag": (), @@ -1060,3 +1063,12 @@ def test_multipart_upload(testdir, tmp_path, base_url, cli): data = base64.b64decode(cassette["http_interactions"][-1]["request"]["body"]["base64_string"]) assert data.startswith(b"file=") # NOTE, that the actual endpoint is not checked in this test + + +@pytest.mark.parametrize("workers", (1, 2)) +@pytest.mark.endpoints("success") +def test_targeted(mocker, cli, cli_args, workers): + target = mocker.spy(hypothesis, "target") + result = cli.run(*cli_args, f"--workers={workers}", "--target=response_time") + assert result.exit_code == ExitCode.OK + target.assert_called_with(mocker.ANY, label="response_time")