feat: A flexible way to select API operations for testing

Ref: #703 #819 #1006

docs/changelog.rst

@@ -7,6 +7,8 @@ Changelog
 **Added**
 
 - ``DataGenerationMethod.all`` shortcut to get all possible enum variants.
+- A flexible way to select API operations for testing. It is now possible to exclude or include them by arbitrary
+  predicates. `#703`_, `#819`_, `#1006`_
 
 `3.10.0`_ - 2021-09-13
 ----------------------
@@ -2188,6 +2190,7 @@ Deprecated
 .. _#1013: https://github.com/schemathesis/schemathesis/issues/1013
 .. _#1010: https://github.com/schemathesis/schemathesis/issues/1010
 .. _#1007: https://github.com/schemathesis/schemathesis/issues/1007
+.. _#1006: https://github.com/schemathesis/schemathesis/issues/1006
 .. _#1003: https://github.com/schemathesis/schemathesis/issues/1003
 .. _#999: https://github.com/schemathesis/schemathesis/issues/999
 .. _#994: https://github.com/schemathesis/schemathesis/issues/994
@@ -2243,6 +2246,7 @@ Deprecated
 .. _#830: https://github.com/schemathesis/schemathesis/issues/830
 .. _#824: https://github.com/schemathesis/schemathesis/issues/824
 .. _#822: https://github.com/schemathesis/schemathesis/issues/822
+.. _#819: https://github.com/schemathesis/schemathesis/issues/819
 .. _#816: https://github.com/schemathesis/schemathesis/issues/816
 .. _#814: https://github.com/schemathesis/schemathesis/issues/814
 .. _#812: https://github.com/schemathesis/schemathesis/issues/812
@@ -2269,6 +2273,7 @@ Deprecated
 .. _#708: https://github.com/schemathesis/schemathesis/issues/708
 .. _#706: https://github.com/schemathesis/schemathesis/issues/706
 .. _#705: https://github.com/schemathesis/schemathesis/issues/705
+.. _#703: https://github.com/schemathesis/schemathesis/issues/703
 .. _#702: https://github.com/schemathesis/schemathesis/issues/702
 .. _#700: https://github.com/schemathesis/schemathesis/issues/700
 .. _#695: https://github.com/schemathesis/schemathesis/issues/695

src/schemathesis/filters.py

@@ -0,0 +1,70 @@
+import enum
+from typing import Any, Callable, List, Optional
+
+import attr
+
+# Indicates the most common place where filters are applied.
+# Other scopes are spec-specific and their filters may be applied earlier to avoid expensive computations
+DEFAULT_SCOPE = None
+
+
+class FilterResult(enum.Enum):
+    """The result of a single filter call.
+
+    This functionality is implemented as a separate enum and not a simple boolean to provide a more descriptive API.
+    """
+
+    INCLUDED = enum.auto()
+    EXCLUDED = enum.auto()
+
+    @property
+    def is_included(self) -> bool:
+        return self == FilterResult.INCLUDED
+
+    @property
+    def is_excluded(self) -> bool:
+        return self == FilterResult.EXCLUDED
+
+    def __bool__(self) -> bool:
+        return self.is_included
+
+    def __and__(self, other: "FilterResult") -> "FilterResult":
+        if self.is_excluded or other.is_excluded:
+            return FilterResult.EXCLUDED
+        return self
+
+
+@attr.s(slots=True)
+class BaseFilter:
+    func: Callable[..., bool] = attr.ib()
+    scope: Optional[str] = attr.ib(default=DEFAULT_SCOPE)
+
+    def apply(self, item: Any) -> FilterResult:
+        raise NotImplementedError
+
+
+@attr.s(slots=True)
+class Include(BaseFilter):
+    def apply(self, item: Any) -> FilterResult:
+        if self.func(item):
+            return FilterResult.INCLUDED
+        return FilterResult.EXCLUDED
+
+
+@attr.s(slots=True)
+class Exclude(BaseFilter):
+    def apply(self, item: Any) -> FilterResult:
+        if self.func(item):
+            return FilterResult.EXCLUDED
+        return FilterResult.INCLUDED
+
+
+def evaluate_filters(filters: List[BaseFilter], item: Any, scope: Optional[str] = DEFAULT_SCOPE) -> FilterResult:
+    """Decide whether the given item passes the filters."""
+    # Lazily apply filters that match the given scope
+    matching_filters = filter(lambda f: f.scope == scope, filters)
+    outcomes = map(lambda f: f.apply(item), matching_filters)
+    # If any filter will exclude the item, then the process short-circuits without evaluating all filters
+    if all(outcomes):
+        return FilterResult.INCLUDED
+    return FilterResult.EXCLUDED

src/schemathesis/hooks.py

@@ -36,7 +36,7 @@ class HookContext:
 
     operation: Optional["APIOperation"] = attr.ib(default=None)  # pragma: no mutate
 
-    @deprecated_property(removed_in="4.0", replacement="operation")
+    @deprecated_property(removed_in="4.0", replacement="`operation`")
     def endpoint(self) -> Optional["APIOperation"]:
         return self.operation
 

src/schemathesis/lazy.py

@@ -22,6 +22,7 @@
     is_given_applied,
     merge_given_args,
     validate_given_args,
+    warn_filtration_arguments,
 )
 
 
@@ -51,6 +52,11 @@ def parametrize(
         data_generation_methods: Union[Iterable[DataGenerationMethod], NotSet] = NOT_SET,
         code_sample_style: Union[str, NotSet] = NOT_SET,
     ) -> Callable:
+        # pylint: disable=too-many-statements
+        for name in ("method", "endpoint", "tag", "operation_id", "skip_deprecated_operations"):
+            value = locals()[name]
+            if value is not NOT_SET:
+                warn_filtration_arguments(name)
         if method is NOT_SET:
             method = self.method
         if endpoint is NOT_SET:

src/schemathesis/models.py

@@ -118,7 +118,7 @@ def __repr__(self) -> str:
                 parts.extend((name, "=", repr(value)))
         return "".join(parts) + ")"
 
-    @deprecated_property(removed_in="4.0", replacement="operation")
+    @deprecated_property(removed_in="4.0", replacement="`operation`")
     def endpoint(self) -> "APIOperation":
         return self.operation
 

src/schemathesis/runner/__init__.py

@@ -31,7 +31,7 @@
 )
 
 
-@deprecated(removed_in="4.0", replacement="schemathesis.runner.from_schema")
+@deprecated(removed_in="4.0", replacement="`schemathesis.runner.from_schema`")
 def prepare(
     schema_uri: Union[str, Dict[str, Any]],
     *,

src/schemathesis/schemas.py

@@ -23,6 +23,7 @@
     Type,
     TypeVar,
     Union,
+    cast,
 )
 from urllib.parse import quote, unquote, urljoin, urlsplit, urlunsplit
 
@@ -34,11 +35,22 @@
 from ._hypothesis import create_test
 from .constants import DEFAULT_DATA_GENERATION_METHODS, CodeSampleStyle, DataGenerationMethod
 from .exceptions import InvalidSchema, UsageError
+from .filters import BaseFilter, Exclude, Include
 from .hooks import HookContext, HookDispatcher, HookScope, dispatch
 from .models import APIOperation, Case
 from .stateful import APIStateMachine, Stateful, StatefulTest
 from .types import Body, Cookies, Filter, FormData, GenericTest, Headers, NotSet, PathParameters, Query
-from .utils import NOT_SET, PARAMETRIZE_MARKER, Err, GenericResponse, GivenInput, Ok, Result, given_proxy
+from .utils import (
+    NOT_SET,
+    PARAMETRIZE_MARKER,
+    Err,
+    GenericResponse,
+    GivenInput,
+    Ok,
+    Result,
+    given_proxy,
+    warn_filtration_arguments,
+)
 
 
 class MethodsDict(CaseInsensitiveDict):
@@ -57,6 +69,7 @@ def __getitem__(self, item: Any) -> Any:
 
 
 C = TypeVar("C", bound=Case)
+S = TypeVar("S", bound="BaseSchema")
 
 
 @attr.s(eq=False)  # pragma: no mutate
@@ -77,6 +90,7 @@ class BaseSchema(Mapping):
         default=DEFAULT_DATA_GENERATION_METHODS
     )  # pragma: no mutate
     code_sample_style: CodeSampleStyle = attr.ib(default=CodeSampleStyle.default())  # pragma: no mutate
+    filters: List[BaseFilter] = attr.ib(factory=list)
 
     def __iter__(self) -> Iterator[str]:
         return iter(self.operations)
@@ -203,6 +217,10 @@ def parametrize(
         _code_sample_style = (
             CodeSampleStyle.from_str(code_sample_style) if isinstance(code_sample_style, str) else code_sample_style
         )
+        for name in ("method", "endpoint", "tag", "operation_id", "skip_deprecated_operations"):
+            value = locals()[name]
+            if value is not NOT_SET:
+                warn_filtration_arguments(name)
 
         def wrapper(func: GenericTest) -> GenericTest:
             if hasattr(func, PARAMETRIZE_MARKER):
@@ -216,7 +234,7 @@ def wrapped_test(*_: Any, **__: Any) -> NoReturn:
 
                 return wrapped_test
             HookDispatcher.add_dispatcher(func)
-            cloned = self.clone(
+            cloned: BaseSchema = self.clone(
                 test_function=func,
                 method=method,
                 endpoint=endpoint,
@@ -251,7 +269,8 @@ def clone(
         skip_deprecated_operations: Union[bool, NotSet] = NOT_SET,
         data_generation_methods: Union[Iterable[DataGenerationMethod], NotSet] = NOT_SET,
         code_sample_style: Union[CodeSampleStyle, NotSet] = NOT_SET,
-    ) -> "BaseSchema":
+        filters: Union[List[BaseFilter], NotSet] = NOT_SET,
+    ) -> S:
         if base_url is NOT_SET:
             base_url = self.base_url
         if method is NOT_SET:
@@ -274,8 +293,13 @@ def clone(
             data_generation_methods = self.data_generation_methods
         if code_sample_style is NOT_SET:
             code_sample_style = self.code_sample_style
+        new_filters = self._construct_filters(
+            endpoint, method, tag, operation_id, cast(bool, skip_deprecated_operations), Include
+        )
+        if filters is not NOT_SET:
+            new_filters += cast(List[BaseFilter], filters)
 
-        return self.__class__(
+        return self.__class__(  # type: ignore
             self.raw_schema,
             location=self.location,
             base_url=base_url,  # type: ignore
@@ -290,8 +314,20 @@ def clone(
             skip_deprecated_operations=skip_deprecated_operations,  # type: ignore
             data_generation_methods=data_generation_methods,  # type: ignore
             code_sample_style=code_sample_style,  # type: ignore
+            filters=new_filters,
         )
 
+    def _construct_filters(
+        self,
+        path: Optional[Filter],
+        method: Optional[Filter],
+        tag: Optional[Filter],
+        operation_id: Optional[Filter],
+        skip_deprecated_operations: Optional[bool],
+        cls: Type[BaseFilter],
+    ) -> List[BaseFilter]:
+        return []
+
     def get_local_hook_dispatcher(self) -> Optional[HookDispatcher]:
         """Get a HookDispatcher instance bound to the test if present."""
         # It might be not present when it is used without pytest via `APIOperation.as_strategy()`
@@ -358,6 +394,18 @@ def validate_response(self, operation: APIOperation, response: GenericResponse)
     def prepare_schema(self, schema: Any) -> Any:
         raise NotImplementedError
 
+    def include_by(self, predicate: Callable) -> "BaseSchema":
+        """Get a new schema that includes API operations that pass the given predicate."""
+        return self._filter_by(Include(predicate))
+
+    def exclude_by(self, predicate: Callable) -> "BaseSchema":
+        """Get a new schema that excludes API operations that pass the given predicate."""
+        return self._filter_by(Exclude(predicate))
+
+    def _filter_by(self, *predicates: BaseFilter) -> S:
+        filters = self.filters + list(predicates)
+        return self.clone(filters=filters)
+
 
 def operations_to_dict(
     operations: Generator[Result[APIOperation, InvalidSchema], None, None]

src/schemathesis/specs/openapi/loaders.py

@@ -16,7 +16,14 @@
 from ...hooks import HookContext, dispatch
 from ...lazy import LazySchema
 from ...types import Filter, NotSet, PathLike
-from ...utils import NOT_SET, StringDatesYAMLLoader, WSGIResponse, require_relative_url, setup_headers
+from ...utils import (
+    NOT_SET,
+    StringDatesYAMLLoader,
+    WSGIResponse,
+    require_relative_url,
+    setup_headers,
+    warn_filtration_arguments,
+)
 from . import definitions
 from .schemas import BaseOpenAPISchema, OpenApi30, SwaggerV20
 
@@ -179,12 +186,18 @@ def from_dict(
 
     :param dict raw_schema: A schema to load.
     """
+    for name in ("method", "endpoint", "tag", "operation_id"):
+        value = locals()[name]
+        if value is not None:
+            warn_filtration_arguments(name)
+    if skip_deprecated_operations is True:
+        warn_filtration_arguments("skip_deprecated_operations")
     _code_sample_style = CodeSampleStyle.from_str(code_sample_style)
     dispatch("before_load_schema", HookContext(), raw_schema)
 
     def init_openapi_2() -> SwaggerV20:
         _maybe_validate_schema(raw_schema, definitions.SWAGGER_20_VALIDATOR, validate_schema)
-        return SwaggerV20(
+        schema = SwaggerV20(
             raw_schema,
             app=app,
             base_url=base_url,
@@ -198,10 +211,11 @@ def init_openapi_2() -> SwaggerV20:
             code_sample_style=_code_sample_style,
             location=location,
         )
+        return schema.include(endpoint, method)
 
     def init_openapi_3() -> OpenApi30:
         _maybe_validate_schema(raw_schema, definitions.OPENAPI_30_VALIDATOR, validate_schema)
-        return OpenApi30(
+        schema = OpenApi30(
             raw_schema,
             app=app,
             base_url=base_url,
@@ -215,6 +229,7 @@ def init_openapi_3() -> OpenApi30:
             code_sample_style=_code_sample_style,
             location=location,
         )
+        return schema.include(endpoint, method)
 
     if force_schema_version == "20":
         return init_openapi_2()