Implement parametrizing tests from pytest fixtures

README.rst

@@ -57,7 +57,6 @@ look like this:
 .. code:: python
 
     # test_api.py
-    import pytest
     import requests
     import schemathesis
 
@@ -154,6 +153,25 @@ With this Swagger schema example, there will be a case with body ``{"name": "Dog
 
 NOTE. Schemathesis supports only examples in ``parameters`` at the moment, examples of individual properties are not supported.
 
+Lazy loading
+~~~~~~~~~~~~
+
+If you have a schema that is not available when the tests are collected, for example it is build with tools
+like ``apispec`` and requires an application instance available, then you can parametrize the tests from a pytest fixture.
+
+.. code:: python
+
+    # test_api.py
+    import schemathesis
+
+    schema = schemathesis.from_pytest_fixture("fixture_name")
+
+    @schema.parametrize()
+    def test_api(case):
+        ...
+
+In this case the test body will be used as a sub-test via ``pytest-subtests`` library.
+
 Documentation
 -------------
 

docs/changelog.rst

@@ -6,6 +6,11 @@ Changelog
 `Unreleased`_
 -------------
 
+Added
+~~~~~
+
+- Parametrizing tests from a pytest fixture via ``pytest-subtests``. `#58`_
+
 Changed
 ~~~~~~~
 
@@ -102,6 +107,7 @@ Fixed
 .. _0.3.0: https://github.com/kiwicom/schemathesis/compare/v0.2.0...v0.3.0
 .. _0.2.0: https://github.com/kiwicom/schemathesis/compare/v0.1.0...v0.2.0
 
+.. _#58: https://github.com/kiwicom/schemathesis/issues/58
 .. _#55: https://github.com/kiwicom/schemathesis/issues/55
 .. _#35: https://github.com/kiwicom/schemathesis/issues/35
 .. _#34: https://github.com/kiwicom/schemathesis/issues/34

pyproject.toml

@@ -31,6 +31,7 @@ hypothesis_jsonschema = "^0.9.7"
 jsonschema = "^3.0.0"
 pytest = ">4.6.4"
 pyyaml = "^5.1"
+pytest-subtests = "^0.2.1"
 
 [tool.poetry.dev-dependencies]
 coverage = "^4.5"
@@ -39,10 +40,10 @@ pytest = ">4.6.4"
 [tool.poetry.plugins]
 pytest11 = {schemathesis = "schemathesis.extra.pytest_plugin"}
 
-[build-system]
-requires = ["poetry>=0.12"]
-build-backend = "poetry.masonry.api"
-
 [tool.black]
 line-length = 120
 target_version = ["py37"]
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

src/schemathesis/__init__.py

@@ -1,2 +1,2 @@
-from .loaders import Parametrizer, from_dict, from_file, from_path, from_uri
+from .loaders import Parametrizer, from_dict, from_file, from_path, from_pytest_fixture, from_uri
 from .models import Case

src/schemathesis/lazy.py

@@ -0,0 +1,48 @@
+from inspect import signature
+from typing import Any, Callable, Dict, Optional
+
+import attr
+from _pytest.fixtures import FixtureRequest
+from pytest_subtests import SubTests
+
+from .schemas import BaseSchema
+from .types import Filter
+
+
+@attr.s(slots=True)
+class LazySchema:
+    fixture_name: str = attr.ib()
+
+    def parametrize(self, filter_method: Optional[Filter] = None, filter_endpoint: Optional[Filter] = None) -> Callable:
+        def wrapper(func: Callable) -> Callable:
+            def test(request: FixtureRequest, subtests: SubTests) -> None:
+                schema = get_schema(request, self.fixture_name, filter_method, filter_endpoint)
+                fixtures = get_fixtures(func, request)
+                node_id = subtests.item._nodeid
+                for endpoint, sub_test in schema.get_all_tests(func):
+                    subtests.item._nodeid = f"{node_id}[{endpoint.method}:{endpoint.path}]"
+                    with subtests.test(method=endpoint.method, path=endpoint.path):
+                        sub_test(**fixtures)
+                subtests.item._nodeid = node_id
+
+            return test
+
+        return wrapper
+
+
+def get_schema(
+    request: FixtureRequest, name: str, filter_method: Optional[Filter] = None, filter_endpoint: Optional[Filter] = None
+) -> BaseSchema:
+    """Loads a schema from the fixture."""
+    schema = request.getfixturevalue(name)
+    if not isinstance(schema, BaseSchema):
+        raise ValueError(f"The given schema must be an instance of BaseSchema, got: {type(schema)}")
+    schema.filter_method = filter_method
+    schema.filter_endpoint = filter_endpoint
+    return schema
+
+
+def get_fixtures(func: Callable, request: FixtureRequest) -> Dict[str, Any]:
+    """Load fixtures, needed for the test function."""
+    sig = signature(func)
+    return {name: request.getfixturevalue(name) for name in sig.parameters if name != "case"}

src/schemathesis/loaders.py

@@ -3,6 +3,7 @@
 
 import yaml
 
+from .lazy import LazySchema
 from .schemas import BaseSchema, OpenApi30, SwaggerV20
 from .types import PathLike
 from .utils import deprecated
@@ -39,6 +40,11 @@ def from_dict(raw_schema: Dict[str, Any]) -> BaseSchema:
     raise ValueError("Unsupported schema type")
 
 
+def from_pytest_fixture(fixture_name: str) -> LazySchema:
+    """Needed for a consistent library API."""
+    return LazySchema(fixture_name)
+
+
 # Backward compatibility
 class Parametrizer:
     from_path = deprecated(from_path, "`Parametrizer.from_path` is deprecated, use `schemathesis.from_path` instead.")

src/schemathesis/schemas.py

@@ -8,12 +8,13 @@
 """
 import itertools
 from copy import deepcopy
-from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Union, overload
+from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Tuple, Union, overload
 from urllib.parse import urljoin
 
 import attr
 import jsonschema
 
+from ._hypothesis import create_test
 from .filters import should_skip_endpoint, should_skip_method
 from .models import Endpoint
 from .types import Filter
@@ -35,6 +36,11 @@ def resolver(self) -> jsonschema.RefResolver:
     def get_all_endpoints(self) -> Generator[Endpoint, None, None]:
         raise NotImplementedError
 
+    def get_all_tests(self, func: Callable) -> Generator[Tuple[Endpoint, Callable], None, None]:
+        """Generate all endpoints and Hypothesis tests for them."""
+        for endpoint in self.get_all_endpoints():
+            yield endpoint, create_test(endpoint, func)
+
     def parametrize(self, filter_method: Optional[Filter] = None, filter_endpoint: Optional[Filter] = None) -> Callable:
         """Mark a test function as a parametrized one."""
 

test/test_lazy.py

@@ -0,0 +1,49 @@
+def test_default(testdir):
+    # When LazySchema is used
+    testdir.make_test(
+        """
+@pytest.fixture
+def simple_schema():
+    return schema
+
+lazy_schema = schemathesis.from_pytest_fixture("simple_schema")
+
+@lazy_schema.parametrize()
+def test_(case):
+    assert case.path == "/v1/users"
+    assert case.method == "GET"
+"""
+    )
+    result = testdir.runpytest("-v")
+    # Then the generated test should use this fixture
+    result.assert_outcomes(passed=1)
+    result.stdout.re_match_lines([r"test_default.py::test_ PASSED", r".*1 passed"])
+
+
+def test_with_fixtures(testdir):
+    # When the test uses custom arguments for pytest fixtures
+    testdir.make_test(
+        """
+@pytest.fixture
+def simple_schema():
+    return schema
+
+lazy_schema = schemathesis.from_pytest_fixture("simple_schema")
+
+@pytest.fixture
+def another():
+    return 1
+
+@lazy_schema.parametrize()
+def test_(request, case, another):
+    request.config.HYPOTHESIS_CASES += 1
+    assert case.path == "/v1/users"
+    assert case.method == "GET"
+    assert another == 1
+"""
+    )
+    result = testdir.runpytest("-v")
+    # Then the generated test should use these fixtures
+    result.assert_outcomes(passed=1)
+    result.stdout.re_match_lines([r"test_with_fixtures.py::test_ PASSED", r".*1 passed"])
+    result.stdout.re_match_lines([r"Hypothesis calls: 1"])