diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/bench.py b/benchmarks/bench.py new file mode 100644 index 0000000..a890137 --- /dev/null +++ b/benchmarks/bench.py @@ -0,0 +1,204 @@ +"""Microbenchmark for the Flagsmith Python SDK's local evaluation hot path. + +Run with:: + + poetry run python -m benchmarks.bench + poetry run python -m benchmarks.bench --profile # cProfile hot path + poetry run python -m benchmarks.bench --iters 20000 # custom iter count + +Mirrors the scenario from issue #198: enable_local_evaluation=True, a +single-identity call into get_identity_flags / get_environment_flags with +~262 features. +""" + +from __future__ import annotations + +import argparse +import cProfile +import json +import pstats +import statistics +import time +from typing import Callable, cast + +from benchmarks.env import build_environment +from flagsmith import Flagsmith +from flagsmith.api.types import EnvironmentModel +from flagsmith.mappers import map_environment_document_to_context + + +def _make_client(n_features: int, with_multivariate: int = 0) -> Flagsmith: + env_doc = cast( + EnvironmentModel, + build_environment( + n_features=n_features, + with_multivariate=with_multivariate, + ), + ) + # Build a local-eval client without hitting the network / starting polling. + # The property setter on ``_evaluation_context`` initialises all the + # derived caches we need; everything else below is what ``Flagsmith.__init__`` + # would otherwise set during its real construction path. + client = Flagsmith.__new__(Flagsmith) + client.offline_mode = False + client.enable_local_evaluation = True + client.offline_handler = None + client.default_flag_handler = None + client.enable_realtime_updates = False + client._analytics_processor = None + client._pipeline_analytics_processor = None + client._environment_updated_at = None + client._evaluation_context = map_environment_document_to_context(env_doc) + return client + + +def _bench( + name: str, + fn: Callable[[], object], + *, + iters: int, + warmup: int, +) -> None: + for _ in range(warmup): + fn() + + samples: list[float] = [] + # Break total iters into batches so we can also report a stdev. + batch_size = max(1, iters // 20) + for _ in range(0, iters, batch_size): + n = min(batch_size, iters) + t0 = time.perf_counter() + for _ in range(n): + fn() + samples.append((time.perf_counter() - t0) / n) + iters -= n + + p50 = statistics.median(samples) * 1e6 + mean = statistics.fmean(samples) * 1e6 + stdev = statistics.pstdev(samples) * 1e6 + print( + f"{name:<32} p50={p50:8.2f} µs mean={mean:8.2f} µs stdev={stdev:7.2f} µs " + f"throughput={1e6 / mean:>10,.0f}/s" + ) + + +def run(iters: int, warmup: int, n_features: int, with_multivariate: int) -> None: + client = _make_client(n_features, with_multivariate=with_multivariate) + traits = {"venue_id": "12345"} + + print( + f"Flagsmith local-eval benchmark | features={n_features} " + f"multivariate={with_multivariate} iters={iters} warmup={warmup}" + ) + _bench( + "get_environment_flags", + client.get_environment_flags, + iters=iters, + warmup=warmup, + ) + _bench( + "get_identity_flags", + lambda: client.get_identity_flags(identifier="anonymous", traits=traits), + iters=iters, + warmup=warmup, + ) + + flags = client.get_identity_flags(identifier="anonymous", traits=traits) + name = next(iter(flags.flags)) + _bench( + "is_feature_enabled (cached)", + lambda: flags.is_feature_enabled(name), + iters=iters * 10, + warmup=warmup, + ) + + +def profile( + iters: int, + n_features: int, + output: str | None, + with_multivariate: int = 0, +) -> None: + client = _make_client(n_features, with_multivariate=with_multivariate) + traits = {"venue_id": "12345"} + + # Warm up JSONPath caches, lru_cache, etc. + for _ in range(200): + client.get_identity_flags(identifier="anonymous", traits=traits) + + profiler = cProfile.Profile() + profiler.enable() + for _ in range(iters): + client.get_identity_flags(identifier="anonymous", traits=traits) + profiler.disable() + + stats = pstats.Stats(profiler).sort_stats(pstats.SortKey.CUMULATIVE) + stats.print_stats(30) + stats.sort_stats(pstats.SortKey.TIME).print_stats(30) + + if output: + profiler.dump_stats(output) + print(f"wrote {output}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--iters", type=int, default=5000) + parser.add_argument("--warmup", type=int, default=500) + parser.add_argument("--features", type=int, default=262) + parser.add_argument( + "--multivariate", + type=int, + default=0, + help="number of features that should have 2-way multivariate variants", + ) + parser.add_argument("--profile", action="store_true") + parser.add_argument("--profile-output", default=None) + parser.add_argument("--json", action="store_true", help="emit JSON summary") + args = parser.parse_args() + + if args.profile: + profile( + args.iters, + args.features, + args.profile_output, + with_multivariate=args.multivariate, + ) + return + + if args.json: + # Alternative machine-readable mode for diffing runs. + client = _make_client(args.features) + traits = {"venue_id": "12345"} + + def _measure(fn: Callable[[], object], count: int) -> float: + for _ in range(args.warmup): + fn() + t0 = time.perf_counter() + for _ in range(count): + fn() + return (time.perf_counter() - t0) / count + + result = { + "features": args.features, + "iters": args.iters, + "get_environment_flags_us": _measure( + client.get_environment_flags, args.iters + ) + * 1e6, + "get_identity_flags_us": _measure( + lambda: client.get_identity_flags( + identifier="anonymous", traits=traits + ), + args.iters, + ) + * 1e6, + } + print(json.dumps(result, indent=2)) + return + + run(args.iters, args.warmup, args.features, args.multivariate) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/env.py b/benchmarks/env.py new file mode 100644 index 0000000..8a9df51 --- /dev/null +++ b/benchmarks/env.py @@ -0,0 +1,56 @@ +"""Synthetic environment builder for local evaluation benchmarks. + +Mirrors the shape of the real environment document (262 features, one segment), +so we can exercise the local eval hot path without needing network access. +""" + +from __future__ import annotations + +import copy +import json +import os +import typing + +DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "tests", "data") +TEMPLATE_PATH = os.path.join(DATA_DIR, "environment.json") + + +def build_environment( + n_features: int = 262, + *, + with_multivariate: int = 0, +) -> dict[str, typing.Any]: + with open(TEMPLATE_PATH) as f: + env: dict[str, typing.Any] = json.load(f) + + # Base feature state to clone. + base_fs = copy.deepcopy(env["feature_states"][0]) + feature_states: list[dict[str, typing.Any]] = [] + for i in range(n_features): + fs = copy.deepcopy(base_fs) + fs["django_id"] = i + 1 + fs["feature"] = { + "name": f"feature_{i:04d}", + "type": "STANDARD", + "id": i + 1, + } + fs["feature_state_value"] = f"value-{i}" + if with_multivariate and i < with_multivariate: + fs["multivariate_feature_state_values"] = [ + { + "multivariate_feature_option": {"value": f"mv-{i}-a"}, + "percentage_allocation": 50.0, + "id": (i + 1) * 100 + 1, + }, + { + "multivariate_feature_option": {"value": f"mv-{i}-b"}, + "percentage_allocation": 50.0, + "id": (i + 1) * 100 + 2, + }, + ] + feature_states.append(fs) + + env["feature_states"] = feature_states + # Strip the (irrelevant) identity override for a clean baseline. + env["identity_overrides"] = [] + return env diff --git a/flagsmith/flagsmith.py b/flagsmith/flagsmith.py index 0ff75fe..571dc1c 100644 --- a/flagsmith/flagsmith.py +++ b/flagsmith/flagsmith.py @@ -117,7 +117,12 @@ def __init__( self._pipeline_analytics_processor: typing.Optional[ PipelineAnalyticsProcessor ] = None - self._evaluation_context: typing.Optional[SDKEvaluationContext] = None + self.__evaluation_context: typing.Optional[SDKEvaluationContext] = None + self._environment_context_without_segments: typing.Optional[ + SDKEvaluationContext + ] = None + self._environment_flags_cache: typing.Optional[Flags] = None + self._identity_flags_match_environment: bool = False self._environment_updated_at: typing.Optional[datetime] = None # argument validation @@ -356,6 +361,46 @@ def update_environment(self) -> None: except (KeyError, TypeError, ValueError): logger.exception("Error parsing environment document") + @property + def _evaluation_context(self) -> typing.Optional[SDKEvaluationContext]: + return self.__evaluation_context + + @_evaluation_context.setter + def _evaluation_context( + self, context: typing.Optional[SDKEvaluationContext] + ) -> None: + """Swap in a new evaluation context and invalidate derived caches. + + Pre-computing the segment-stripped view and a couple of shape flags + once per refresh keeps the hot path for ``get_environment_flags`` + and the short-circuited ``get_identity_flags`` allocation-free. + """ + self.__evaluation_context = context + self._environment_flags_cache = None + if context is None: + self._environment_context_without_segments = None + self._identity_flags_match_environment = False + return + context_without_segments = context.copy() + context_without_segments.pop("segments", None) + self._environment_context_without_segments = context_without_segments + # An identity's flags only differ from the environment's when either + # a feature has variants (percentage split) or some segment can + # override feature values (segment overrides / identity overrides). + # When neither is true we can skip the per-call engine evaluation + # entirely and reuse the cached environment Flags. + has_variants = any( + feature.get("variants") + for feature in (context.get("features") or {}).values() + ) + has_segment_overrides = any( + segment.get("overrides") + for segment in (context.get("segments") or {}).values() + ) + self._identity_flags_match_environment = not ( + has_variants or has_segment_overrides + ) + def _get_headers( self, environment_key: str, @@ -375,24 +420,25 @@ def _get_headers( return headers def _get_environment_flags_from_document(self) -> Flags: - if self._evaluation_context is None: - raise TypeError("No environment present") + if (cached := self._environment_flags_cache) is not None: + return cached - # Omit segments from evaluation context for environment flags - # as they are only relevant for identity-specific evaluations - context_without_segments = self._evaluation_context.copy() - context_without_segments.pop("segments", None) + context_without_segments = self._environment_context_without_segments + if context_without_segments is None: + raise TypeError("No environment present") evaluation_result = engine.get_evaluation_result( context=context_without_segments, ) - return Flags.from_evaluation_result( + flags = Flags.from_evaluation_result( evaluation_result=evaluation_result, analytics_processor=self._analytics_processor, default_flag_handler=self.default_flag_handler, pipeline_analytics_processor=self._pipeline_analytics_processor, ) + self._environment_flags_cache = flags + return flags def _get_identity_flags_from_document( self, @@ -402,15 +448,26 @@ def _get_identity_flags_from_document( if self._evaluation_context is None: raise TypeError("No environment present") + if self._identity_flags_match_environment: + # Reuse the cached environment Flag dict but wrap it in a fresh + # ``Flags`` carrying this identity's metadata, so pipeline analytics + # still see per-identity events. + env_flags = self._get_environment_flags_from_document() + return Flags( + flags=env_flags.flags, + default_flag_handler=self.default_flag_handler, + _analytics_processor=self._analytics_processor, + _pipeline_analytics_processor=self._pipeline_analytics_processor, + _identity_identifier=identifier, + _traits=resolve_trait_values(traits), + ) + context = map_context_and_identity_data_to_context( context=self._evaluation_context, identifier=identifier, traits=traits, ) - evaluation_result = engine.get_evaluation_result( - context=context, - ) - + evaluation_result = engine.get_evaluation_result(context=context) return Flags.from_evaluation_result( evaluation_result=evaluation_result, analytics_processor=self._analytics_processor, diff --git a/flagsmith/mappers.py b/flagsmith/mappers.py index 3ed3e3d..eba050d 100644 --- a/flagsmith/mappers.py +++ b/flagsmith/mappers.py @@ -90,10 +90,13 @@ def map_context_and_identity_data_to_context( identifier: str, traits: typing.Optional[TraitMapping] = None, ) -> SDKEvaluationContext: + # Pre-computing ``identity.key`` here lets the engine skip the context + # copy it otherwise does in ``get_enriched_context`` on every call. return { **context, "identity": { "identifier": identifier, + "key": f"{context['environment']['key']}_{identifier}", "traits": resolve_trait_values(traits) or {}, }, } @@ -245,23 +248,35 @@ def _map_environment_document_feature_states_to_feature_contexts( if multivariate_feature_state_values := feature_state.get( "multivariate_feature_state_values" ): - feature_context["variants"] = [ - { - "value": multivariate_feature_state_value[ - "multivariate_feature_option" - ]["value"], - "weight": multivariate_feature_state_value["percentage_allocation"], - "priority": ( - multivariate_feature_state_value.get("id") - or uuid.UUID( - multivariate_feature_state_value["mv_fs_value_uuid"] - ).int - ), - } - for multivariate_feature_state_value in multivariate_feature_state_values - ] + # Pre-sort by priority once at env load so evaluation doesn't + # have to sort on every identity call. + feature_context["variants"] = sorted( + ( + { + "value": multivariate_feature_state_value[ + "multivariate_feature_option" + ]["value"], + "weight": multivariate_feature_state_value[ + "percentage_allocation" + ], + "priority": ( + multivariate_feature_state_value.get("id") + or uuid.UUID( + multivariate_feature_state_value["mv_fs_value_uuid"] + ).int + ), + } + for multivariate_feature_state_value in multivariate_feature_state_values + ), + key=_variant_priority, + ) if feature_segment := feature_state.get("feature_segment"): feature_context["priority"] = feature_segment["priority"] yield feature_context + + +def _variant_priority(variant: typing.Mapping[str, typing.Any]) -> int: + priority: int = variant["priority"] + return priority diff --git a/flagsmith/models.py b/flagsmith/models.py index 8d3765c..731e74c 100644 --- a/flagsmith/models.py +++ b/flagsmith/models.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys import typing from dataclasses import dataclass, field @@ -7,19 +8,24 @@ from flagsmith.exceptions import FlagsmithFeatureDoesNotExistError from flagsmith.types import SDKEvaluationResult, SDKFlagResult +# dataclass(slots=True) is only available on 3.10+; we still support 3.9. +_DATACLASS_KWARGS: typing.Dict[str, bool] = ( + {"slots": True} if sys.version_info >= (3, 10) else {} +) -@dataclass + +@dataclass(**_DATACLASS_KWARGS) class BaseFlag: enabled: bool value: typing.Union[str, int, float, bool, None] -@dataclass +@dataclass(**_DATACLASS_KWARGS) class DefaultFlag(BaseFlag): is_default: bool = field(default=True) -@dataclass +@dataclass(**_DATACLASS_KWARGS) class Flag(BaseFlag): feature_id: int feature_name: str @@ -73,12 +79,19 @@ def from_evaluation_result( identity_identifier: typing.Optional[str] = None, traits: typing.Optional[typing.Dict[str, typing.Any]] = None, ) -> Flags: + # Inlined Flag construction: local eval always produces metadata so + # we can skip the per-flag helper call and its redundant branching. + flag_cls = Flag + flags: typing.Dict[str, Flag] = {} + for flag_name, flag_result in evaluation_result["flags"].items(): + flags[flag_name] = flag_cls( + enabled=flag_result["enabled"], + value=flag_result["value"], + feature_name=flag_name, + feature_id=flag_result["metadata"]["id"], + ) return cls( - flags={ - flag_name: flag - for flag_name, flag_result in evaluation_result["flags"].items() - if (flag := Flag.from_evaluation_result(flag_result)) - }, + flags=flags, default_flag_handler=default_flag_handler, _analytics_processor=analytics_processor, _pipeline_analytics_processor=pipeline_analytics_processor, diff --git a/tests/test_flagsmith.py b/tests/test_flagsmith.py index b0ad097..f53d53c 100644 --- a/tests/test_flagsmith.py +++ b/tests/test_flagsmith.py @@ -184,84 +184,45 @@ def test_get_identity_flags_calls_api_when_no_local_environment_with_traits( assert identity_flags.all_flags()[0].feature_name == "some_feature" -@responses.activate() def test_get_identity_flags_uses_local_environment_when_available( flagsmith: Flagsmith, evaluation_context: SDKEvaluationContext, - mocker: MockerFixture, ) -> None: # Given flagsmith._evaluation_context = evaluation_context flagsmith.enable_local_evaluation = True - mock_engine = mocker.patch("flagsmith.flagsmith.engine") - expected_evaluation_result = { - "flags": { - "some_feature": { - "name": "some_feature", - "enabled": True, - "value": "some-feature-state-value", - "metadata": {"id": 1}, - } - }, - "segments": [], - } - - identifier = "identifier" - traits = {"some_trait": "some_value"} - - mock_engine.get_evaluation_result.return_value = expected_evaluation_result - - # When - identity_flags = flagsmith.get_identity_flags(identifier, traits).all_flags() + # When: non-overridden identity gets the environment-level flag value. + default_flags = flagsmith.get_identity_flags( + "identifier", + traits={"some_trait": "some_value"}, + ).all_flags() # Then - mock_engine.get_evaluation_result.assert_called_once() - call_args = mock_engine.get_evaluation_result.call_args - context = call_args[1]["context"] - assert context["identity"]["identifier"] == identifier - assert context["identity"]["traits"]["some_trait"] == "some_value" - assert "some_trait" in context["identity"]["traits"] - - assert identity_flags[0].enabled is True - assert identity_flags[0].value == "some-feature-state-value" + assert default_flags[0].enabled is True + assert default_flags[0].value == "some-value" + assert default_flags[0].feature_name == "some_feature" -def test_get_identity_flags_includes_segments_in_evaluation_context( - mocker: MockerFixture, +def test_get_identity_flags_applies_identity_overrides( local_eval_flagsmith: Flagsmith, ) -> None: - # Given - mock_get_evaluation_result = mocker.patch( - "flagsmith.flagsmith.engine.get_evaluation_result", - autospec=True, - ) - - expected_evaluation_result = { - "flags": { - "some_feature": { - "name": "some_feature", - "enabled": True, - "value": "some-feature-state-value", - "metadata": {"id": 1}, - } - }, - "segments": [], - } - - identifier = "identifier" - traits = {"some_trait": "some_value"} - - mock_get_evaluation_result.return_value = expected_evaluation_result + # Given the fixture env.json declares an identity override for "overridden-id". # When - local_eval_flagsmith.get_identity_flags(identifier, traits) + identity_flags = local_eval_flagsmith.get_identity_flags( + "overridden-id" + ).all_flags() - # Then - # Verify segments are present in the context passed to the engine for identity flags - call_args = mock_get_evaluation_result.call_args - context = call_args[1]["context"] - assert "segments" in context + # Then: the override wins over the base feature value. + assert identity_flags[0].feature_name == "some_feature" + assert identity_flags[0].enabled is False + assert identity_flags[0].value == "some-overridden-value" + + # And a non-overridden identity still gets the base value. + default_flags = local_eval_flagsmith.get_identity_flags("other-id").all_flags() + assert default_flags[0].enabled is True + assert default_flags[0].value == "some-value" @responses.activate()