diff --git a/doc/changelog.rst b/doc/changelog.rst index a603838..afb87c9 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,13 @@ Changelog ========= +[0.1.15] - Unreleased +--------------------- + +Added +^^^^^ +- Check filtering. :issue:`23` + [0.1.14] - 2025-03-28 --------------------- diff --git a/doc/tutorial.rst b/doc/tutorial.rst index fb4179a..2f6d9d5 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -36,6 +36,73 @@ and pass it to the :meth:`~scim2_tester.check_server` method: print(result.status.name, result.title) +scim2-tester supports filtering tests using tags and resource types to run only specific subsets of checks. +This is useful for targeted testing or when developing specific features. + +Tag-based filtering +~~~~~~~~~~~~~~~~~~~ + +Tests are organized using hierarchical tags that allow fine-grained control over which checks to execute. Use the :paramref:`~scim2_tester.check_server.include_tags` and :paramref:`~scim2_tester.check_server.exclude_tags` parameters: + +.. code-block:: python + + from scim2_tester import check_server + + # Run only discovery tests + results = check_server(client, include_tags={"discovery"}) + + # Run all CRUD operations except delete + results = check_server( + client, + include_tags={"crud"}, + exclude_tags={"crud:delete"} + ) + + # Run only resource creation tests + results = check_server(client, include_tags={"crud:create"}) + +Available tags include: + +- **discovery**: Configuration endpoint tests (:class:`~scim2_models.ServiceProviderConfig`, :class:`~scim2_models.ResourceType`, :class:`~scim2_models.Schema`) +- **service-provider-config**: :class:`~scim2_models.ServiceProviderConfig` endpoint tests +- **resource-types**: :class:`~scim2_models.ResourceType` endpoint tests +- **schemas**: :class:`~scim2_models.Schema` endpoint tests +- **crud**: All CRUD operation tests +- **crud:create**: Resource creation tests +- **crud:read**: Resource reading tests +- **crud:update**: Resource update tests +- **crud:delete**: Resource deletion tests +- **misc**: Miscellaneous tests + +The tag system is hierarchical, so ``crud`` will match ``crud:create``, ``crud:read``, etc. + +Resource type filtering +~~~~~~~~~~~~~~~~~~~~~~~ + +You can also filter tests by resource type using the :paramref:`~scim2_tester.check_server.resource_types` parameter: + +.. code-block:: python + + # Test only User resources + results = check_server(client, resource_types=["User"]) + + # Test both User and Group resources + results = check_server(client, resource_types=["User", "Group"]) + +Combining filters +~~~~~~~~~~~~~~~~~ + +Filters can be combined for precise control using both :paramref:`~scim2_tester.check_server.include_tags` and :paramref:`~scim2_tester.check_server.resource_types` parameters: + +.. code-block:: python + + # Test only User creation and reading + results = check_server( + client, + include_tags={"crud:create", "crud:read"}, + resource_types=["User"] + ) + Unit test suite integration =========================== @@ -61,3 +128,29 @@ As :class:`~scim2_client.engines.werkzeug.TestSCIMClient` relies on :doc:`Werkze testclient = Client(app) client = TestSCIMClient(app=testclient, scim_prefix="/scim/v2") check_server(client, raise_exceptions=True) + +Parametrized testing +~~~~~~~~~~~~~~~~~~~~ + +For comprehensive test coverage, you can create parametrized tests that exercise different combinations of tags and resource types using :func:`~scim2_tester.discovery.get_all_available_tags` and :func:`~scim2_tester.discovery.get_standard_resource_types`: + +.. code-block:: python + + import pytest + from scim2_tester import Status, check_server + from scim2_tester.discovery import get_all_available_tags, get_standard_resource_types + + @pytest.mark.parametrize("tag", get_all_available_tags()) + @pytest.mark.parametrize("resource_type", [None] + get_standard_resource_types()) + def test_individual_filters(scim_client, tag, resource_type): + results = check_server( + scim_client, + raise_exceptions=False, + include_tags={tag}, + resource_types=resource_type + ) + + for result in results: + assert result.status in (Status.SKIPPED, Status.SUCCESS) + +This parametrized approach automatically discovers all available tags and resource types, ensuring that your test suite covers all possible combinations as your SCIM implementation evolves. Each test verifies that results have either :attr:`~scim2_tester.Status.SUCCESS` or :attr:`~scim2_tester.Status.SKIPPED` status. diff --git a/scim2_tester/__init__.py b/scim2_tester/__init__.py index d1d471b..4b93d62 100644 --- a/scim2_tester/__init__.py +++ b/scim2_tester/__init__.py @@ -1,7 +1,17 @@ from .checker import check_server +from .discovery import get_all_available_tags +from .discovery import get_standard_resource_types from .utils import CheckConfig from .utils import CheckResult from .utils import SCIMTesterError from .utils import Status -__all__ = ["check_server", "Status", "CheckResult", "CheckConfig", "SCIMTesterError"] +__all__ = [ + "check_server", + "Status", + "CheckResult", + "CheckConfig", + "SCIMTesterError", + "get_all_available_tags", + "get_standard_resource_types", +] diff --git a/scim2_tester/checker.py b/scim2_tester/checker.py index 09fcb0e..6e1a954 100644 --- a/scim2_tester/checker.py +++ b/scim2_tester/checker.py @@ -14,7 +14,7 @@ from scim2_tester.utils import checker -@checker +@checker("misc") def check_random_url(conf: CheckConfig) -> CheckResult: """Check that a request to a random URL returns a 404 Error object.""" probably_invalid_url = f"/{str(uuid.uuid4())}" @@ -44,7 +44,13 @@ def check_random_url(conf: CheckConfig) -> CheckResult: ) -def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult]: +def check_server( + client: SCIMClient, + raise_exceptions=False, + include_tags: set[str] | None = None, + exclude_tags: set[str] | None = None, + resource_types: list[str] | None = None, +) -> list[CheckResult]: """Perform a series of check to a SCIM server. It starts by retrieving the standard :class:`~scim2_models.ServiceProviderConfig`, @@ -56,27 +62,74 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult :param client: A SCIM client that will perform the requests. :param raise_exceptions: Whether exceptions should be raised or stored in a :class:`~scim2_tester.CheckResult` object. + :param include_tags: Execute only checks with at least one of these tags. + :param exclude_tags: Skip checks with any of these tags. + :param resource_types: Filter by resource type names (e.g., ["User", "Group"]). + + Available tags: + - **discovery**: Tests for configuration endpoints (ServiceProviderConfig, ResourceTypes, Schemas) + - **service-provider-config**: Tests for ServiceProviderConfig endpoint + - **resource-types**: Tests for ResourceTypes endpoint + - **schemas**: Tests for Schemas endpoint + - **crud**: All CRUD operation tests + - **crud:create**: Resource creation tests + - **crud:read**: Resource reading tests + - **crud:update**: Resource update tests + - **crud:delete**: Resource deletion tests + - **misc**: Miscellaneous tests (e.g., random URL access) + + Example usage:: + + # Run only discovery tests + results = check_server(client, include_tags={"discovery"}) + + # Run CRUD tests except delete operations + results = check_server( + client, include_tags={"crud"}, exclude_tags={"crud:delete"} + ) + + # Test only User resources + results = check_server(client, resource_types=["User"]) + + # Test only User creation and reading + results = check_server( + client, include_tags={"crud:create", "crud:read"}, resource_types=["User"] + ) """ - conf = CheckConfig(client, raise_exceptions) + conf = CheckConfig( + client=client, + raise_exceptions=raise_exceptions, + include_tags=include_tags, + exclude_tags=exclude_tags, + resource_types=resource_types, + ) results = [] # Get the initial basic objects result_spc = check_service_provider_config_endpoint(conf) results.append(result_spc) - if not conf.client.service_provider_config: + if result_spc.status != Status.SKIPPED and not conf.client.service_provider_config: conf.client.service_provider_config = result_spc.data results_resource_types = check_resource_types_endpoint(conf) results.extend(results_resource_types) if not conf.client.resource_types: - conf.client.resource_types = results_resource_types[0].data + # Find first non-skipped result with data + for rt_result in results_resource_types: + if rt_result.status != Status.SKIPPED and rt_result.data: + conf.client.resource_types = rt_result.data + break results_schemas = check_schemas_endpoint(conf) results.extend(results_schemas) if not conf.client.resource_models: - conf.client.resource_models = conf.client.build_resource_models( - conf.client.resource_types or [], results_schemas[0].data or [] - ) + # Find first non-skipped result with data + for schema_result in results_schemas: + if schema_result.status != Status.SKIPPED and schema_result.data: + conf.client.resource_models = conf.client.build_resource_models( + conf.client.resource_types or [], schema_result.data or [] + ) + break if ( not conf.client.service_provider_config @@ -91,7 +144,15 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult # Resource checks for resource_type in conf.client.resource_types or []: - results.extend(check_resource_type(conf, resource_type)) + # Filter by resource type if specified + if conf.resource_types and resource_type.name not in conf.resource_types: + continue + + resource_results = check_resource_type(conf, resource_type) + # Add resource type to each result for better tracking + for result in resource_results: + result.resource_type = resource_type.name + results.extend(resource_results) return results @@ -100,10 +161,28 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult from httpx import Client from scim2_client.engines.httpx import SyncSCIMClient - parser = argparse.ArgumentParser(description="Process some integers.") + parser = argparse.ArgumentParser(description="SCIM server compliance checker.") parser.add_argument("host") parser.add_argument("--token", required=False) parser.add_argument("--verbose", required=False, action="store_true") + parser.add_argument( + "--include-tags", + nargs="+", + help="Run only checks with these tags", + required=False, + ) + parser.add_argument( + "--exclude-tags", + nargs="+", + help="Skip checks with these tags", + required=False, + ) + parser.add_argument( + "--resource-types", + nargs="+", + help="Filter by resource type names", + required=False, + ) args = parser.parse_args() client = Client( @@ -112,9 +191,24 @@ def check_server(client: SCIMClient, raise_exceptions=False) -> list[CheckResult ) scim = SyncSCIMClient(client) scim.discover() - results = check_server(scim) + + include_tags: set[str] | None = ( + set(args.include_tags) if args.include_tags else None + ) + exclude_tags: set[str] | None = ( + set(args.exclude_tags) if args.exclude_tags else None + ) + + results = check_server( + scim, + include_tags=include_tags, + exclude_tags=exclude_tags, + resource_types=args.resource_types, + ) + for result in results: - print(result.status.name, result.title) + resource_info = f" [{result.resource_type}]" if result.resource_type else "" + print(f"{result.status.name} {result.title}{resource_info}") if result.reason: print(" ", result.reason) if args.verbose and result.data: diff --git a/scim2_tester/discovery.py b/scim2_tester/discovery.py new file mode 100644 index 0000000..6d941d0 --- /dev/null +++ b/scim2_tester/discovery.py @@ -0,0 +1,47 @@ +"""Utility functions for discovering available tags and resources.""" + +from scim2_tester.utils import get_registered_tags + + +def get_all_available_tags() -> set[str]: + """Get all available tags from the global registry. + + This function returns tags that have been registered by checker decorators + throughout the codebase. The registration happens automatically when + modules containing @checker decorators are imported. + + :returns: Set of all unique tags found in the codebase. + :rtype: set[str] + """ + # Import all scim2_tester modules to ensure decorators are executed + _ensure_modules_imported() + + # Get registered tags from the global registry + registered_tags = get_registered_tags() + + return registered_tags + + +def _ensure_modules_imported() -> None: + """Ensure all scim2_tester modules are imported to register their tags.""" + # Import key modules that contain checker decorators + try: + import scim2_tester.checker # noqa: F401 + import scim2_tester.resource_delete # noqa: F401 + import scim2_tester.resource_get # noqa: F401 + import scim2_tester.resource_post # noqa: F401 + import scim2_tester.resource_put # noqa: F401 + import scim2_tester.resource_types # noqa: F401 + import scim2_tester.schemas # noqa: F401 + import scim2_tester.service_provider_config # noqa: F401 + except ImportError: + pass # In case some modules don't exist or have import issues + + +def get_standard_resource_types() -> list[str]: + """Get standard SCIM resource types. + + :returns: List of standard resource type names. + :rtype: list[str] + """ + return ["User", "Group"] diff --git a/scim2_tester/filling.py b/scim2_tester/filling.py index 124422c..ab50574 100644 --- a/scim2_tester/filling.py +++ b/scim2_tester/filling.py @@ -36,7 +36,7 @@ def create_minimal_object( def model_from_ref_type( - conf: CheckConfig, ref_type: type, different_than: Resource + conf: CheckConfig, ref_type: type, different_than: type[Resource] ) -> type[Resource]: """Return "User" from "Union[Literal['User'], Literal['Group']]".""" diff --git a/scim2_tester/resource.py b/scim2_tester/resource.py index ebb5c3b..050159a 100644 --- a/scim2_tester/resource.py +++ b/scim2_tester/resource.py @@ -29,6 +29,9 @@ def check_resource_type( results = [] garbages = [] + created_obj = None + + # Always try to create an object - the decorator will decide if it should be skipped field_names = [ field_name for field_name in model.model_fields.keys() @@ -38,17 +41,26 @@ def check_resource_type( obj, obj_garbages = fill_with_random_values(conf, model(), field_names) garbages += obj_garbages - result = check_object_creation(conf, obj) - results.append(result) + create_result = check_object_creation(conf, obj) + + # Only add to results if creation was explicitly requested (not skipped) + if create_result.status != Status.SKIPPED: + results.append(create_result) - if result.status == Status.SUCCESS: - created_obj = result.data - result = check_object_query(conf, created_obj) - results.append(result) + # If creation succeeded (either explicitly or as dependency), we have an object + if create_result.status == Status.SUCCESS: + created_obj = create_result.data - result = check_object_query_without_id(conf, created_obj) - results.append(result) + # Try read operations - decorator will skip if not needed + read_result = check_object_query(conf, created_obj) + if read_result.status != Status.SKIPPED: + results.append(read_result) + read_without_id_result = check_object_query_without_id(conf, created_obj) + if read_without_id_result.status != Status.SKIPPED: + results.append(read_without_id_result) + + # Try update operations - decorator will skip if not needed field_names = [ field_name for field_name in model.model_fields.keys() @@ -57,12 +69,17 @@ def check_resource_type( ] _, obj_garbages = fill_with_random_values(conf, created_obj, field_names) garbages += obj_garbages - result = check_object_replacement(conf, created_obj) - results.append(result) - result = check_object_deletion(conf, created_obj) - results.append(result) + update_result = check_object_replacement(conf, created_obj) + if update_result.status != Status.SKIPPED: + results.append(update_result) + + # Try delete operations - decorator will skip if not needed + delete_result = check_object_deletion(conf, created_obj) + if delete_result.status != Status.SKIPPED: + results.append(delete_result) + # Cleanup remaining garbage for garbage in reversed(garbages): conf.client.delete(garbage) diff --git a/scim2_tester/resource_delete.py b/scim2_tester/resource_delete.py index 69d9f27..924fa6f 100644 --- a/scim2_tester/resource_delete.py +++ b/scim2_tester/resource_delete.py @@ -6,7 +6,7 @@ from scim2_tester.utils import checker -@checker +@checker("crud:delete") def check_object_deletion(conf: CheckConfig, obj: Resource) -> CheckResult: """Perform an object deletion.""" conf.client.delete( diff --git a/scim2_tester/resource_get.py b/scim2_tester/resource_get.py index 70e4189..c5cea78 100644 --- a/scim2_tester/resource_get.py +++ b/scim2_tester/resource_get.py @@ -26,7 +26,7 @@ def model_from_resource_type( return None -@checker +@checker("crud:read") def check_object_query(conf: CheckConfig, obj: Resource) -> CheckResult: """Perform an object query by knowing its id. @@ -46,7 +46,7 @@ def check_object_query(conf: CheckConfig, obj: Resource) -> CheckResult: ) -@checker +@checker("crud:read") def check_object_query_without_id(conf: CheckConfig, obj: Resource) -> CheckResult: """Perform the query of all objects of one kind. diff --git a/scim2_tester/resource_post.py b/scim2_tester/resource_post.py index 8ed8b1f..003d912 100644 --- a/scim2_tester/resource_post.py +++ b/scim2_tester/resource_post.py @@ -6,7 +6,7 @@ from scim2_tester.utils import checker -@checker +@checker("crud:create") def check_object_creation(conf: CheckConfig, obj: Resource) -> CheckResult: """Perform an object creation. diff --git a/scim2_tester/resource_put.py b/scim2_tester/resource_put.py index d9cbb8e..7ca8af8 100644 --- a/scim2_tester/resource_put.py +++ b/scim2_tester/resource_put.py @@ -6,7 +6,7 @@ from scim2_tester.utils import checker -@checker +@checker("crud:update") def check_object_replacement(conf: CheckConfig, obj: Resource) -> CheckResult: """Perform an object replacement. diff --git a/scim2_tester/resource_types.py b/scim2_tester/resource_types.py index 8f70c64..723253e 100644 --- a/scim2_tester/resource_types.py +++ b/scim2_tester/resource_types.py @@ -32,7 +32,7 @@ def check_resource_types_endpoint(conf: CheckConfig) -> list[CheckResult]: return results -@checker +@checker("discovery", "resource-types") def check_query_all_resource_types(conf: CheckConfig) -> CheckResult: response = conf.client.query( ResourceType, expected_status_codes=conf.expected_status_codes or [200] @@ -44,7 +44,7 @@ def check_query_all_resource_types(conf: CheckConfig) -> CheckResult: ) -@checker +@checker("discovery", "resource-types") def check_query_resource_type_by_id( conf: CheckConfig, resource_type: ResourceType ) -> CheckResult: @@ -62,7 +62,7 @@ def check_query_resource_type_by_id( return CheckResult(conf, status=Status.SUCCESS, reason=reason, data=response) -@checker +@checker("discovery", "resource-types") def check_access_invalid_resource_type(conf: CheckConfig) -> CheckResult: probably_invalid_id = str(uuid.uuid4()) response = conf.client.query( diff --git a/scim2_tester/schemas.py b/scim2_tester/schemas.py index 6ff54ee..bea29a7 100644 --- a/scim2_tester/schemas.py +++ b/scim2_tester/schemas.py @@ -32,7 +32,7 @@ def check_schemas_endpoint(conf: CheckConfig) -> list[CheckResult]: return results -@checker +@checker("discovery", "schemas") def check_query_all_schemas(conf: CheckConfig) -> CheckResult: response = conf.client.query( Schema, expected_status_codes=conf.expected_status_codes or [200] @@ -46,7 +46,7 @@ def check_query_all_schemas(conf: CheckConfig) -> CheckResult: ) -@checker +@checker("discovery", "schemas") def check_query_schema_by_id(conf: CheckConfig, schema: Schema) -> CheckResult: response = conf.client.query( Schema, @@ -62,7 +62,7 @@ def check_query_schema_by_id(conf: CheckConfig, schema: Schema) -> CheckResult: return CheckResult(conf, status=Status.SUCCESS, reason=reason, data=response) -@checker +@checker("discovery", "schemas") def check_access_invalid_schema(conf: CheckConfig) -> CheckResult: probably_invalid_id = str(uuid.uuid4()) response = conf.client.query( diff --git a/scim2_tester/service_provider_config.py b/scim2_tester/service_provider_config.py index ad6bfcc..e1532ac 100644 --- a/scim2_tester/service_provider_config.py +++ b/scim2_tester/service_provider_config.py @@ -6,7 +6,7 @@ from .utils import checker -@checker +@checker("discovery", "service-provider-config") def check_service_provider_config_endpoint( conf: CheckConfig, ) -> CheckResult: diff --git a/scim2_tester/utils.py b/scim2_tester/utils.py index 3ac5991..fe23412 100644 --- a/scim2_tester/utils.py +++ b/scim2_tester/utils.py @@ -1,5 +1,6 @@ import functools from dataclasses import dataclass +from dataclasses import field from enum import Enum from enum import auto from typing import Any @@ -7,10 +8,47 @@ from scim2_client import SCIMClient from scim2_client import SCIMClientError +# Global registry for all tags discovered by checker decorators +_REGISTERED_TAGS: set[str] = set() + + +def get_registered_tags() -> set[str]: + """Get all tags that have been registered by checker decorators. + + :returns: Set of all registered tags. + :rtype: set[str] + """ + return _REGISTERED_TAGS.copy() + + +def _matches_hierarchical_tags(func_tags: set[str], filter_tags: set[str]) -> bool: + """Check if function tags match filter tags using hierarchical logic. + + Supports patterns like: + - "crud" matches "crud:read", "crud:create", etc. + - "crud:read" matches exactly "crud:read" + + :param func_tags: Tags on the function + :param filter_tags: Tags to filter by + :returns: True if there's a match + """ + for filter_tag in filter_tags: + for func_tag in func_tags: + # Exact match + if filter_tag == func_tag: + return True + + # Hierarchical match: filter "crud" matches func "crud:read" + if ":" in func_tag and filter_tag in func_tag.split(":"): + return True + + return False + class Status(Enum): SUCCESS = auto() ERROR = auto() + SKIPPED = auto() @dataclass @@ -26,6 +64,15 @@ class CheckConfig: expected_status_codes: list[int] | None = None """The expected response status codes.""" + include_tags: set[str] | None = None + """Execute only checks with at least one of these tags.""" + + exclude_tags: set[str] | None = None + """Skip checks with any of these tags.""" + + resource_types: list[str] | None = None + """Filter by resource type names (e.g., ["User", "Group"]).""" + class SCIMTesterError(Exception): """Exception raised when a check failed and the `raise_exceptions` config parameter is :data:`True`.""" @@ -58,38 +105,98 @@ class CheckResult: data: Any | None = None """Any related data that can help to debug.""" + tags: set[str] = field(default_factory=set) + """Tags associated with this check for filtering.""" + + resource_type: str | None = None + """The resource type name if this check is related to a specific resource.""" + def __post_init__(self): if self.conf.raise_exceptions and self.status == Status.ERROR: raise SCIMTesterError(self.reason, self) -def checker(func): - """Decorate checker methods. +def checker(*tags): + """Decorate checker methods with tags for selective execution. - It adds a title and a description to the returned result, extracted from the method name and its docstring. - It catches SCIMClient errors. + - It allows tagging checks for selective execution. + - It skips execution based on tag filtering in CheckConfig. + + Usage: + @checker("discovery", "schemas") + def check_schemas_endpoint(...): + ... """ - @functools.wraps(func) - def wrapped(conf: CheckConfig, *args, **kwargs): - try: - result = func(conf, *args, **kwargs) - except SCIMClientError as exc: - if conf.raise_exceptions: - raise - - reason = f"{exc} {exc.__cause__}" if exc.__cause__ else str(exc) - result = CheckResult( - conf, status=Status.ERROR, reason=reason, data=exc.source - ) - - # decorate results - if isinstance(result, CheckResult): - result.title = func.__name__ - result.description = func.__doc__ - else: - result[0].title = func.__name__ - result[0].description = func.__doc__ - return result - - return wrapped + def decorator(func): + @functools.wraps(func) + def wrapped(conf: CheckConfig, *args, **kwargs): + func_tags = set(tags) if tags else set() + + # Check if function should be skipped based on tag filtering + if conf.include_tags and not _matches_hierarchical_tags( + func_tags, conf.include_tags + ): + return CheckResult( + conf, + status=Status.SKIPPED, + title=func.__name__, + description=func.__doc__, + tags=func_tags, + reason="Skipped due to tag filtering", + ) + + if conf.exclude_tags and _matches_hierarchical_tags( + func_tags, conf.exclude_tags + ): + return CheckResult( + conf, + status=Status.SKIPPED, + title=func.__name__, + description=func.__doc__, + tags=func_tags, + reason="Skipped due to tag exclusion", + ) + + # Execute the function normally + try: + result = func(conf, *args, **kwargs) + except SCIMClientError as exc: + if conf.raise_exceptions: + raise + + reason = f"{exc} {exc.__cause__}" if exc.__cause__ else str(exc) + result = CheckResult( + conf, status=Status.ERROR, reason=reason, data=exc.source + ) + + # decorate results + if isinstance(result, CheckResult): + result.title = func.__name__ + result.description = func.__doc__ + result.tags = func_tags + elif isinstance(result, list): + # Handle list of results + for r in result: + r.title = func.__name__ + r.description = func.__doc__ + r.tags = func_tags + return result + + # Store tags on the function for potential filtering + wrapped.tags = set(tags) if tags else set() + + # Register tags globally for discovery + if tags: + _REGISTERED_TAGS.update(tags) + + return wrapped + + # Handle both @checker and @checker("tag1", "tag2") + if len(tags) == 1 and callable(tags[0]): + func = tags[0] + tags = () + return decorator(func) + return decorator diff --git a/tests/test_scim2_server.py b/tests/test_scim2_server.py index 87f82a5..db80205 100644 --- a/tests/test_scim2_server.py +++ b/tests/test_scim2_server.py @@ -6,7 +6,10 @@ from scim2_server.utils import load_default_schemas from werkzeug.test import Client +from scim2_tester import Status from scim2_tester import check_server +from scim2_tester.discovery import get_all_available_tags +from scim2_tester.discovery import get_standard_resource_types @pytest.fixture @@ -24,11 +27,118 @@ def scim2_server(): def test_discovered_scim2_server(scim2_server): + """Test the complete SCIM server with discovery.""" client = TestSCIMClient(Client(scim2_server)) client.discover() - check_server(client, raise_exceptions=True) + results = check_server(client, raise_exceptions=False) + + # Verify all tests executed successfully (no skipped) + executed_results = [r for r in results if r.status != Status.SKIPPED] + assert len(executed_results) > 0 + assert all(r.status in (Status.SUCCESS, Status.ERROR) for r in executed_results) def test_undiscovered_scim2_server(scim2_server): + """Test the SCIM server without initial discovery.""" client = TestSCIMClient(Client(scim2_server)) - check_server(client, raise_exceptions=True) + results = check_server(client, raise_exceptions=False) + + # Verify all tests executed successfully (no skipped) + executed_results = [r for r in results if r.status != Status.SKIPPED] + assert len(executed_results) > 0 + assert all(r.status in (Status.SUCCESS, Status.ERROR) for r in executed_results) + + +@pytest.mark.parametrize("tag", get_all_available_tags()) +@pytest.mark.parametrize("resource_type", [None] + get_standard_resource_types()) +def test_individual_filters(scim2_server, tag, resource_type): + """Test individual tags and resource types.""" + client = TestSCIMClient(Client(scim2_server)) + client.discover() + results = check_server( + client, raise_exceptions=False, include_tags={tag}, resource_types=resource_type + ) + for result in results: + assert result.status in (Status.SKIPPED, Status.SUCCESS), ( + f"Result {result.title} failed: {result.reason}" + ) + + +def test_filtering_functionality(scim2_server): + """Test that filtering produces different result sets.""" + client = TestSCIMClient(Client(scim2_server)) + client.discover() + + # Get all results + all_results = check_server(client, raise_exceptions=False) + all_executed = [r for r in all_results if r.status != Status.SKIPPED] + + # Test discovery only + discovery_results = check_server( + client, raise_exceptions=False, include_tags={"discovery"} + ) + discovery_executed = [r for r in discovery_results if r.status != Status.SKIPPED] + discovery_skipped = [r for r in discovery_results if r.status == Status.SKIPPED] + + # Should have some executed discovery results + assert len(discovery_executed) > 0, "Expected some executed discovery results" + + # Should have some skipped results (non-discovery) + assert len(discovery_skipped) > 0, "Expected some skipped non-discovery results" + + # Should have fewer results when filtering + assert len(discovery_executed) < len(all_executed), ( + "Expected fewer results when filtering" + ) + + # Test misc only + misc_results = check_server(client, raise_exceptions=False, include_tags={"misc"}) + misc_executed = [r for r in misc_results if r.status != Status.SKIPPED] + + # Should have at least one misc result + assert len(misc_executed) > 0, "Expected at least one misc result" + + # Should have fewer misc results than all results + assert len(misc_executed) < len(all_executed), ( + "Expected fewer misc results than all results" + ) + + +def test_tag_discovery_utility(scim2_server): + """Test that the tag discovery utility works correctly.""" + discovered_tags = get_all_available_tags() + + # Should discover the hierarchical core tags + core_tags = { + "discovery", + "crud:create", + "crud:read", + "crud:update", + "crud:delete", + "misc", + "service-provider-config", + "resource-types", + "schemas", + } + discovered_core = discovered_tags.intersection(core_tags) + assert len(discovered_core) >= 8, ( + f"Expected at least 8 core tags, got {len(discovered_core)}: {discovered_core}" + ) + + # Test with actual server - only test discoverable function-level tags + client = TestSCIMClient(Client(scim2_server)) + client.discover() + + # Test function-level tags + function_level_tags = { + "discovery", + "service-provider-config", + "resource-types", + "schemas", + "crud:create", + "misc", + } + for tag in function_level_tags.intersection(discovered_tags): + results = check_server(client, raise_exceptions=False, include_tags={tag}) + executed = [r for r in results if r.status != Status.SKIPPED] + assert len(executed) > 0, f"Tag '{tag}' produced no executed results" diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..2beeb3d --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,114 @@ +import pytest + +from scim2_tester.utils import CheckConfig +from scim2_tester.utils import CheckResult +from scim2_tester.utils import SCIMTesterError +from scim2_tester.utils import Status +from scim2_tester.utils import checker + + +def test_checker_decorator_with_tags(): + """Test the checker decorator with tags.""" + + @checker("tag1", "tag2") + def check_function(conf: CheckConfig) -> CheckResult: + """Test check function.""" + return CheckResult(conf, status=Status.SUCCESS, reason="Success") + + # Verify function has tags attribute + assert hasattr(check_function, "tags") + assert check_function.tags == {"tag1", "tag2"} + + # Test that result gets tags + conf = CheckConfig(client=None, raise_exceptions=False) + result = check_function(conf) + assert result.tags == {"tag1", "tag2"} + assert result.title == "check_function" + assert result.description == "Test check function." + + +def test_checker_decorator_without_tags(): + """Test the checker decorator without tags.""" + + @checker + def check_function(conf: CheckConfig) -> CheckResult: + """Test check function.""" + return CheckResult(conf, status=Status.SUCCESS, reason="Success") + + # Verify function has empty tags + assert hasattr(check_function, "tags") + assert check_function.tags == set() + + # Test that result gets empty tags + conf = CheckConfig(client=None, raise_exceptions=False) + result = check_function(conf) + assert result.tags == set() + + +def test_checker_decorator_with_list_results(): + """Test the checker decorator with list of results.""" + + @checker("tag1") + def check_function(conf: CheckConfig) -> list[CheckResult]: + """Test check function.""" + return [ + CheckResult(conf, status=Status.SUCCESS, reason="Success 1"), + CheckResult(conf, status=Status.SUCCESS, reason="Success 2"), + ] + + conf = CheckConfig(client=None, raise_exceptions=False) + results = check_function(conf) + + # All results should have tags + assert all(r.tags == {"tag1"} for r in results) + assert all(r.title == "check_function" for r in results) + assert all(r.description == "Test check function." for r in results) + + +def test_scim_tester_error(): + """Test SCIMTesterError exception.""" + conf = CheckConfig(client=None, raise_exceptions=False) + error = SCIMTesterError("Test error", conf) + assert str(error) == "Test error" + assert error.conf == conf + + +def test_check_result_raises_exception_when_configured(): + """Test that CheckResult raises exception when configured.""" + conf = CheckConfig(client=None, raise_exceptions=True) + + with pytest.raises(SCIMTesterError) as exc_info: + CheckResult(conf, status=Status.ERROR, reason="Test error") + + assert str(exc_info.value) == "Test error" + + +def test_check_result_no_exception_when_not_configured(): + """Test that CheckResult doesn't raise exception when not configured.""" + conf = CheckConfig(client=None, raise_exceptions=False) + result = CheckResult(conf, status=Status.ERROR, reason="Test error") + assert result.status == Status.ERROR + assert result.reason == "Test error" + + +def test_check_result_no_exception_on_success(): + """Test that CheckResult doesn't raise exception on success.""" + conf = CheckConfig(client=None, raise_exceptions=True) + result = CheckResult(conf, status=Status.SUCCESS, reason="Success") + assert result.status == Status.SUCCESS + assert result.reason == "Success" + + +def test_check_result_default_values(): + """Test CheckResult default values.""" + conf = CheckConfig(client=None, raise_exceptions=False) + result = CheckResult(conf, status=Status.SUCCESS) + + assert result.conf == conf + assert result.status == Status.SUCCESS + assert result.title is None + assert result.description is None + assert result.reason is None + assert result.data is None + assert result.tags == set() + assert result.resource_type is None