Implement check filtering

doc/changelog.rst

@@ -1,6 +1,13 @@
 Changelog
 =========
 
+[0.1.15] - Unreleased
+---------------------
+
+Added
+^^^^^
+- Check filtering. :issue:`23`
+
 [0.1.14] - 2025-03-28
 ---------------------
 

doc/tutorial.rst

@@ -36,6 +36,73 @@ and pass it to the :meth:`~scim2_tester.check_server` method:
         print(result.status.name, result.title)
 
 
+scim2-tester supports filtering tests using tags and resource types to run only specific subsets of checks.
+This is useful for targeted testing or when developing specific features.
+
+Tag-based filtering
+~~~~~~~~~~~~~~~~~~~
+
+Tests are organized using hierarchical tags that allow fine-grained control over which checks to execute. Use the :paramref:`~scim2_tester.check_server.include_tags` and :paramref:`~scim2_tester.check_server.exclude_tags` parameters:
+
+.. code-block:: python
+
+    from scim2_tester import check_server
+
+    # Run only discovery tests
+    results = check_server(client, include_tags={"discovery"})
+
+    # Run all CRUD operations except delete
+    results = check_server(
+        client,
+        include_tags={"crud"},
+        exclude_tags={"crud:delete"}
+    )
+
+    # Run only resource creation tests
+    results = check_server(client, include_tags={"crud:create"})
+
+Available tags include:
+
+- **discovery**: Configuration endpoint tests (:class:`~scim2_models.ServiceProviderConfig`, :class:`~scim2_models.ResourceType`, :class:`~scim2_models.Schema`)
+- **service-provider-config**: :class:`~scim2_models.ServiceProviderConfig` endpoint tests
+- **resource-types**: :class:`~scim2_models.ResourceType` endpoint tests
+- **schemas**: :class:`~scim2_models.Schema` endpoint tests
+- **crud**: All CRUD operation tests
+- **crud:create**: Resource creation tests
+- **crud:read**: Resource reading tests
+- **crud:update**: Resource update tests
+- **crud:delete**: Resource deletion tests
+- **misc**: Miscellaneous tests
+
+The tag system is hierarchical, so ``crud`` will match ``crud:create``, ``crud:read``, etc.
+
+Resource type filtering
+~~~~~~~~~~~~~~~~~~~~~~~
+
+You can also filter tests by resource type using the :paramref:`~scim2_tester.check_server.resource_types` parameter:
+
+.. code-block:: python
+
+    # Test only User resources
+    results = check_server(client, resource_types=["User"])
+
+    # Test both User and Group resources
+    results = check_server(client, resource_types=["User", "Group"])
+
+Combining filters
+~~~~~~~~~~~~~~~~~
+
+Filters can be combined for precise control using both :paramref:`~scim2_tester.check_server.include_tags` and :paramref:`~scim2_tester.check_server.resource_types` parameters:
+
+.. code-block:: python
+
+    # Test only User creation and reading
+    results = check_server(
+        client,
+        include_tags={"crud:create", "crud:read"},
+        resource_types=["User"]
+    )
+
 Unit test suite integration
 ===========================
 
@@ -61,3 +128,29 @@ As :class:`~scim2_client.engines.werkzeug.TestSCIMClient` relies on :doc:`Werkze
         testclient = Client(app)
         client = TestSCIMClient(app=testclient, scim_prefix="/scim/v2")
         check_server(client, raise_exceptions=True)
+
+Parametrized testing
+~~~~~~~~~~~~~~~~~~~~
+
+For comprehensive test coverage, you can create parametrized tests that exercise different combinations of tags and resource types using :func:`~scim2_tester.discovery.get_all_available_tags` and :func:`~scim2_tester.discovery.get_standard_resource_types`:
+
+.. code-block:: python
+
+    import pytest
+    from scim2_tester import Status, check_server
+    from scim2_tester.discovery import get_all_available_tags, get_standard_resource_types
+
+    @pytest.mark.parametrize("tag", get_all_available_tags())
+    @pytest.mark.parametrize("resource_type", [None] + get_standard_resource_types())
+    def test_individual_filters(scim_client, tag, resource_type):
+        results = check_server(
+            scim_client,
+            raise_exceptions=False,
+            include_tags={tag},
+            resource_types=resource_type
+        )
+
+        for result in results:
+            assert result.status in (Status.SKIPPED, Status.SUCCESS)
+
+This parametrized approach automatically discovers all available tags and resource types, ensuring that your test suite covers all possible combinations as your SCIM implementation evolves. Each test verifies that results have either :attr:`~scim2_tester.Status.SUCCESS` or :attr:`~scim2_tester.Status.SKIPPED` status.

scim2_tester/__init__.py

@@ -1,7 +1,17 @@
 from .checker import check_server
+from .discovery import get_all_available_tags
+from .discovery import get_standard_resource_types
 from .utils import CheckConfig
 from .utils import CheckResult
 from .utils import SCIMTesterError
 from .utils import Status
 
-__all__ = ["check_server", "Status", "CheckResult", "CheckConfig", "SCIMTesterError"]
+__all__ = [
+    "check_server",
+    "Status",
+    "CheckResult",
+    "CheckConfig",
+    "SCIMTesterError",
+    "get_all_available_tags",
+    "get_standard_resource_types",
+]

scim2_tester/checker.py

@@ -14,7 +14,7 @@
 from scim2_tester.utils import checker
 
 
-@checker
+@checker("misc")
 def check_random_url(conf: CheckConfig) -> CheckResult:
     """Check that a request to a random URL returns a 404 Error object."""
     probably_invalid_url = f"/{str(uuid.uuid4())}"
@@ -44,7 +44,13 @@ def check_random_url(conf: CheckConfig) -> CheckResult:
     )
 
 
-def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult]:
+def check_server(
+    client: SCIMClient,
+    raise_exceptions=False,
+    include_tags: set[str] | None = None,
+    exclude_tags: set[str] | None = None,
+    resource_types: list[str] | None = None,
+) -> list[CheckResult]:
     """Perform a series of check to a SCIM server.
 
     It starts by retrieving the standard :class:`~scim2_models.ServiceProviderConfig`,
@@ -56,27 +62,74 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult
 
     :param client: A SCIM client that will perform the requests.
     :param raise_exceptions: Whether exceptions should be raised or stored in a :class:`~scim2_tester.CheckResult` object.
+    :param include_tags: Execute only checks with at least one of these tags.
+    :param exclude_tags: Skip checks with any of these tags.
+    :param resource_types: Filter by resource type names (e.g., ["User", "Group"]).
+
+    Available tags:
+        - **discovery**: Tests for configuration endpoints (ServiceProviderConfig, ResourceTypes, Schemas)
+        - **service-provider-config**: Tests for ServiceProviderConfig endpoint
+        - **resource-types**: Tests for ResourceTypes endpoint
+        - **schemas**: Tests for Schemas endpoint
+        - **crud**: All CRUD operation tests
+        - **crud:create**: Resource creation tests
+        - **crud:read**: Resource reading tests
+        - **crud:update**: Resource update tests
+        - **crud:delete**: Resource deletion tests
+        - **misc**: Miscellaneous tests (e.g., random URL access)
+
+    Example usage::
+
+        # Run only discovery tests
+        results = check_server(client, include_tags={"discovery"})
+
+        # Run CRUD tests except delete operations
+        results = check_server(
+            client, include_tags={"crud"}, exclude_tags={"crud:delete"}
+        )
+
+        # Test only User resources
+        results = check_server(client, resource_types=["User"])
+
+        # Test only User creation and reading
+        results = check_server(
+            client, include_tags={"crud:create", "crud:read"}, resource_types=["User"]
+        )
     """
-    conf = CheckConfig(client, raise_exceptions)
+    conf = CheckConfig(
+        client=client,
+        raise_exceptions=raise_exceptions,
+        include_tags=include_tags,
+        exclude_tags=exclude_tags,
+        resource_types=resource_types,
+    )
     results = []
 
     # Get the initial basic objects
     result_spc = check_service_provider_config_endpoint(conf)
     results.append(result_spc)
-    if not conf.client.service_provider_config:
+    if result_spc.status != Status.SKIPPED and not conf.client.service_provider_config:
         conf.client.service_provider_config = result_spc.data
 
     results_resource_types = check_resource_types_endpoint(conf)
     results.extend(results_resource_types)
     if not conf.client.resource_types:
-        conf.client.resource_types = results_resource_types[0].data
+        # Find first non-skipped result with data
+        for rt_result in results_resource_types:
+            if rt_result.status != Status.SKIPPED and rt_result.data:
+                conf.client.resource_types = rt_result.data
+                break
 
     results_schemas = check_schemas_endpoint(conf)
     results.extend(results_schemas)
     if not conf.client.resource_models:
-        conf.client.resource_models = conf.client.build_resource_models(
-            conf.client.resource_types or [], results_schemas[0].data or []
-        )
+        # Find first non-skipped result with data
+        for schema_result in results_schemas:
+            if schema_result.status != Status.SKIPPED and schema_result.data:
+                conf.client.resource_models = conf.client.build_resource_models(
+                    conf.client.resource_types or [], schema_result.data or []
+                )
+                break
 
     if (
         not conf.client.service_provider_config
@@ -91,7 +144,15 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult
 
     # Resource checks
     for resource_type in conf.client.resource_types or []:
-        results.extend(check_resource_type(conf, resource_type))
+        # Filter by resource type if specified
+        if conf.resource_types and resource_type.name not in conf.resource_types:
+            continue
+
+        resource_results = check_resource_type(conf, resource_type)
+        # Add resource type to each result for better tracking
+        for result in resource_results:
+            result.resource_type = resource_type.name
+        results.extend(resource_results)
 
     return results
 
@@ -100,10 +161,28 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult
     from httpx import Client
     from scim2_client.engines.httpx import SyncSCIMClient
 
-    parser = argparse.ArgumentParser(description="Process some integers.")
+    parser = argparse.ArgumentParser(description="SCIM server compliance checker.")
     parser.add_argument("host")
     parser.add_argument("--token", required=False)
     parser.add_argument("--verbose", required=False, action="store_true")
+    parser.add_argument(
+        "--include-tags",
+        nargs="+",
+        help="Run only checks with these tags",
+        required=False,
+    )
+    parser.add_argument(
+        "--exclude-tags",
+        nargs="+",
+        help="Skip checks with these tags",
+        required=False,
+    )
+    parser.add_argument(
+        "--resource-types",
+        nargs="+",
+        help="Filter by resource type names",
+        required=False,
+    )
     args = parser.parse_args()
 
     client = Client(
@@ -112,9 +191,24 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult
     )
     scim = SyncSCIMClient(client)
     scim.discover()
-    results = check_server(scim)
+
+    include_tags: set[str] | None = (
+        set(args.include_tags) if args.include_tags else None
+    )
+    exclude_tags: set[str] | None = (
+        set(args.exclude_tags) if args.exclude_tags else None
+    )
+
+    results = check_server(
+        scim,
+        include_tags=include_tags,
+        exclude_tags=exclude_tags,
+        resource_types=args.resource_types,
+    )
+
     for result in results:
-        print(result.status.name, result.title)
+        resource_info = f" [{result.resource_type}]" if result.resource_type else ""
+        print(f"{result.status.name} {result.title}{resource_info}")
         if result.reason:
             print("  ", result.reason)
             if args.verbose and result.data:

scim2_tester/discovery.py

@@ -0,0 +1,47 @@
+"""Utility functions for discovering available tags and resources."""
+
+from scim2_tester.utils import get_registered_tags
+
+
+def get_all_available_tags() -> set[str]:
+    """Get all available tags from the global registry.
+
+    This function returns tags that have been registered by checker decorators
+    throughout the codebase. The registration happens automatically when
+    modules containing @checker decorators are imported.
+
+    :returns: Set of all unique tags found in the codebase.
+    :rtype: set[str]
+    """
+    # Import all scim2_tester modules to ensure decorators are executed
+    _ensure_modules_imported()
+
+    # Get registered tags from the global registry
+    registered_tags = get_registered_tags()
+
+    return registered_tags
+
+
+def _ensure_modules_imported() -> None:
+    """Ensure all scim2_tester modules are imported to register their tags."""
+    # Import key modules that contain checker decorators
+    try:
+        import scim2_tester.checker  # noqa: F401
+        import scim2_tester.resource_delete  # noqa: F401
+        import scim2_tester.resource_get  # noqa: F401
+        import scim2_tester.resource_post  # noqa: F401
+        import scim2_tester.resource_put  # noqa: F401
+        import scim2_tester.resource_types  # noqa: F401
+        import scim2_tester.schemas  # noqa: F401
+        import scim2_tester.service_provider_config  # noqa: F401
+    except ImportError:
+        pass  # In case some modules don't exist or have import issues
+
+
+def get_standard_resource_types() -> list[str]:
+    """Get standard SCIM resource types.
+
+    :returns: List of standard resource type names.
+    :rtype: list[str]
+    """
+    return ["User", "Group"]

scim2_tester/filling.py

@@ -36,7 +36,7 @@ def create_minimal_object(
 
 
 def model_from_ref_type(
-    conf: CheckConfig, ref_type: type, different_than: Resource
+    conf: CheckConfig, ref_type: type, different_than: type[Resource]
 ) -> type[Resource]:
     """Return "User" from "Union[Literal['User'], Literal['Group']]"."""