fix: Not raising an error when tests generated by schemas loaded with…

… `from_pytest_fixture` match no API operations

Ref: #1342

docs/changelog.rst

@@ -7,6 +7,7 @@ Changelog
 **Fixed**
 
 - Silently failing to detect numeric status codes when the schema contains a shared ``parameters`` key. `#1343`_
+- Not raising an error when tests generated by schemas loaded with ``from_pytest_fixture`` match no API operations. `#1342`_
 
 `3.11.2`_ - 2021-11-30
 ----------------------
@@ -2205,6 +2206,7 @@ Deprecated
 .. _0.2.0: https://github.com/schemathesis/schemathesis/compare/v0.1.0...v0.2.0
 
 .. _#1343: https://github.com/schemathesis/schemathesis/issues/1343
+.. _#1342: https://github.com/schemathesis/schemathesis/issues/1342
 .. _#1340: https://github.com/schemathesis/schemathesis/issues/1340
 .. _#1336: https://github.com/schemathesis/schemathesis/issues/1336
 .. _#1331: https://github.com/schemathesis/schemathesis/issues/1331

src/schemathesis/extra/pytest_plugin.py

@@ -19,6 +19,7 @@
     PARAMETRIZE_MARKER,
     Ok,
     Result,
+    fail_on_no_matches,
     get_given_args,
     get_given_kwargs,
     is_given_applied,
@@ -184,9 +185,7 @@ def collect(self) -> List[Function]:  # type: ignore
                 for item in self._gen_items(operation, data_generation_method)
             ]
             if not items:
-                pytest.fail(
-                    f"Test function {self.nodeid} does not match any API operations and therefore has no effect"
-                )
+                fail_on_no_matches(self.nodeid)
             return items
         except Exception:
             pytest.fail("Error during collection")

src/schemathesis/lazy.py

@@ -16,6 +16,7 @@
     NOT_SET,
     GivenInput,
     Ok,
+    fail_on_no_matches,
     get_given_args,
     get_given_kwargs,
     given_proxy,
@@ -106,6 +107,8 @@ def test(request: FixtureRequest) -> None:
                 node_id = request.node._nodeid
                 settings = getattr(test, "_hypothesis_internal_use_settings", None)
                 tests = list(schema.get_all_tests(func, settings, _given_kwargs=given_kwargs))
+                if not tests:
+                    fail_on_no_matches(node_id)
                 request.session.testscollected += len(tests)
                 capmam = request.node.config.pluginmanager.get_plugin("capturemanager")
                 if capmam is not None:

src/schemathesis/utils.py

@@ -25,6 +25,7 @@
     overload,
 )
 
+import pytest
 import requests
 import yaml
 import yarl
@@ -93,6 +94,10 @@ def is_schemathesis_test(func: Callable) -> bool:
         return False
 
 
+def fail_on_no_matches(node_id: str) -> NoReturn:  # type: ignore
+    pytest.fail(f"Test function {node_id} does not match any API operations and therefore has no effect")
+
+
 def force_tuple(item: Filter) -> Union[List, Set, Tuple]:
     if not isinstance(item, (list, set, tuple)):
         return (item,)

test/test_filters.py

@@ -213,7 +213,7 @@ def test_(request, case):
     result.assert_outcomes(**{key: 1})
     result.stdout.re_match_lines(
         [
-            r"E   Failed: Test function test_error_on_no_matches.py::test_ does not "
+            r"E *Failed: Test function test_error_on_no_matches.py::test_ does not "
             r"match any API operations and therefore has no effect"
         ]
     )

test/test_lazy.py

@@ -1,6 +1,7 @@
 import pytest
 
 from schemathesis import DataGenerationMethod
+from schemathesis.constants import IS_PYTEST_ABOVE_54
 
 
 def test_default(testdir):
@@ -661,3 +662,30 @@ def test_(case):
     # Then all hooks should be merged
     result = testdir.runpytest("-v")
     result.assert_outcomes(passed=1)
+
+
+def test_error_on_no_matches(testdir):
+    # When test filters don't match any operation
+    testdir.make_test(
+        """
+@pytest.fixture()
+def api_schema():
+    return schemathesis.from_dict(raw_schema)
+
+lazy_schema = schemathesis.from_pytest_fixture("api_schema")
+
+@lazy_schema.parametrize(operation_id=["does-not-exist"])
+@settings(max_examples=1)
+def test_(case):
+    pass
+""",
+    )
+    result = testdir.runpytest("-v")
+    # Then it should be a failure (the collection phase is done, so it can't be an error)
+    result.assert_outcomes(failed=1)
+    result.stdout.re_match_lines(
+        [
+            r"E *Failed: Test function test_error_on_no_matches.py::test_ does not "
+            r"match any API operations and therefore has no effect"
+        ]
+    )