From c8f12f6767c9949acef970fa0f2281d803f2ab55 Mon Sep 17 00:00:00 2001 From: IIIIbntttt Date: Thu, 6 Aug 2026 11:19:44 +0400 Subject: [PATCH 1/2] test: add direct-call baseline and threaded-contention benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close the two gaps left open in #84 after the CodSpeed suite landed: - test_baseline_direct_call and its async twin measure the unwrapped callable, so any call-path result divides into an overhead ratio — the number a prospective user actually wants. - benchmarks/test_contention.py drives one closed breaker from four worker threads to track the work under the shared lock as a trend. The docstring and CONTRIBUTING.md spell out that instruction counting runs threads one at a time: the number is lock-path work, not wall-clock contention. --- CHANGELOG.md | 13 ++++++++ CONTRIBUTING.md | 8 +++++ benchmarks/test_call_paths.py | 10 ++++++ benchmarks/test_contention.py | 57 +++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) create mode 100644 benchmarks/test_contention.py diff --git a/CHANGELOG.md b/CHANGELOG.md index da9efd7..4705b24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Direct-call baseline and threaded-contention benchmarks**, closing out the + two gaps left in #84. The CodSpeed suite reported the absolute cost of every + protected path but not the number a prospective user actually wants — the + breaker's overhead relative to calling the function directly. + `test_baseline_direct_call` and its async twin make that ratio derivable + from any run. `benchmarks/test_contention.py` drives one breaker from four + worker threads so the work under the shared lock is tracked as a trend; its + docstring and `CONTRIBUTING.md` spell out that instruction counting runs + threads one at a time, so the number is lock-path work, not wall-clock + contention. + ## [2.1.4] - 2026-08-03 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9c4357..2e0c999 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,6 +153,14 @@ Every pull request runs the same suite through Add a benchmark when you touch a hot path: the call paths, the state machine, the sliding windows or the pipeline. +`test_baseline_direct_call` (and its async twin) measure the unwrapped +callable — divide any call-path result by the matching baseline to read the +breaker's overhead as a ratio rather than an absolute count. The contention +benchmark (`benchmarks/test_contention.py`) carries a caveat: CodSpeed counts +instructions under Valgrind, which runs threads one at a time, so it tracks +the work done under the breaker's lock when many threads call through it — not +the wall-clock cost of real contention. + ## Expectations - **Tests first.** New behaviour and bug fixes come with tests; the suite keeps diff --git a/benchmarks/test_call_paths.py b/benchmarks/test_call_paths.py index 325f5b2..a37aa1e 100644 --- a/benchmarks/test_call_paths.py +++ b/benchmarks/test_call_paths.py @@ -40,6 +40,16 @@ def runner() -> Iterator[asyncio.Runner]: yield loop_runner +def test_baseline_direct_call(benchmark: BenchmarkFixture) -> None: + """The unwrapped sync callable: the denominator for every overhead ratio.""" + benchmark(lambda: _work(1, 2)) + + +def test_baseline_direct_call_async(benchmark: BenchmarkFixture, runner: asyncio.Runner) -> None: + """The unwrapped coroutine on the same pre-built loop the async paths pay for.""" + benchmark(lambda: runner.run(_async_work(1, 2))) + + def test_call_sync(benchmark: BenchmarkFixture) -> None: """``call`` on a closed breaker: detect, admit, run, classify, record.""" breaker = CircuitBreaker(name='bench-call-sync', config=_CLOSED_CONFIG) diff --git a/benchmarks/test_contention.py b/benchmarks/test_contention.py new file mode 100644 index 0000000..14b7689 --- /dev/null +++ b/benchmarks/test_contention.py @@ -0,0 +1,57 @@ +"""Benchmark for modest threaded contention around one shared breaker. + +Every protected call takes the breaker's single ``threading.Lock`` twice — +once to admit, once to record the outcome — so calls from several threads all +funnel through it. This benchmark drives one closed breaker from four worker +threads so a change in the work done under that lock shows up as a trend. + +Read the number with care: CodSpeed's simulation mode counts CPU instructions +under Valgrind, which runs threads one at a time. What is measured is the +instruction cost of the shared lock path when calls arrive from many threads — +not the waiting, scheduling or cache traffic real contention adds to +wall-clock time. Behaviour under real contention is asserted by +``tests/test_concurrency.py``, including on free-threaded CPython. +""" + +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor + +import pytest +from pytest_codspeed import BenchmarkFixture + +from interlock import CircuitBreaker, Config, State + +_THREADS = 4 +_CALLS_PER_THREAD = 25 +_CLOSED_CONFIG = Config(window_size=100, minimum_number_of_calls=10) + + +def _work(left: int, right: int) -> int: + """The protected payload: cheap enough to expose the breaker's own cost.""" + return left + right + + +@pytest.fixture +def executor() -> Iterator[ThreadPoolExecutor]: + """A worker pool reused across iterations, so thread startup is excluded.""" + with ThreadPoolExecutor(max_workers=_THREADS) as pool: + yield pool + + +def test_contended_calls(benchmark: BenchmarkFixture, executor: ThreadPoolExecutor) -> None: + """Four threads sharing one closed breaker: the lock path under contention.""" + breaker = CircuitBreaker(name='bench-contended', config=_CLOSED_CONFIG) + + def hammer() -> int: + total = 0 + for _ in range(_CALLS_PER_THREAD): + total += breaker.call(_work, 1, 2) + return total + + def contended_round() -> int: + futures = [executor.submit(hammer) for _ in range(_THREADS)] + return sum(future.result() for future in futures) + + assert contended_round() == _THREADS * _CALLS_PER_THREAD * 3 + assert breaker.state is State.CLOSED + benchmark(contended_round) From 84fffba1aa1d51303ae126e8915853ff42b14a1a Mon Sep 17 00:00:00 2001 From: IIIIbntttt Date: Fri, 7 Aug 2026 12:12:32 +0400 Subject: [PATCH 2/2] test: give the async baseline the guarded paths' wrapper frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The breaker path drives a wrapper coroutine that awaits breaker.call, while the baseline ran the bare coroutine — so the overhead ratio charged the breaker for one coroutine frame real usage pays on both sides. Wrapping the baseline the same way makes the ratio isolate the breaker itself. --- benchmarks/test_call_paths.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/test_call_paths.py b/benchmarks/test_call_paths.py index a37aa1e..0e3ba5b 100644 --- a/benchmarks/test_call_paths.py +++ b/benchmarks/test_call_paths.py @@ -46,8 +46,17 @@ def test_baseline_direct_call(benchmark: BenchmarkFixture) -> None: def test_baseline_direct_call_async(benchmark: BenchmarkFixture, runner: asyncio.Runner) -> None: - """The unwrapped coroutine on the same pre-built loop the async paths pay for.""" - benchmark(lambda: runner.run(_async_work(1, 2))) + """The unwrapped coroutine, behind the same wrapper frame the guarded paths pay. + + Real callers await the protected work from inside their own coroutine either + way, so the baseline wraps ``_async_work`` exactly like ``test_call_async`` + wraps ``breaker.call`` — the ratio then isolates the breaker, not a frame. + """ + + async def bare() -> int: + return await _async_work(1, 2) + + benchmark(lambda: runner.run(bare())) def test_call_sync(benchmark: BenchmarkFixture) -> None: