From 342e902c4c4f2bea0fed6f9196d40cd20f34ae1b Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Thu, 7 May 2026 11:15:44 -0500 Subject: [PATCH 1/2] fix: measure individual call durations in sequential phase of concurrency decorator The old approach timed the entire sequential loop (including event loop scheduling overhead between iterations), while the concurrent phase used asyncio.gather which has a fast-path with minimal inter-task overhead. This asymmetry inflated the sequential/concurrent ratio, causing false positives in concurrency gain detection. Sum per-iteration durations instead, excluding event loop scheduling overhead from the measurement. --- codeflash/code_utils/codeflash_wrap_decorator.py | 8 +++++--- codeflash/code_utils/instrument_existing_tests.py | 5 +++-- pyproject.toml | 6 +++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/codeflash/code_utils/codeflash_wrap_decorator.py b/codeflash/code_utils/codeflash_wrap_decorator.py index a6b6d339f..dad501d3c 100644 --- a/codeflash/code_utils/codeflash_wrap_decorator.py +++ b/codeflash/code_utils/codeflash_wrap_decorator.py @@ -181,13 +181,15 @@ async def async_wrapper(*args: Any, **kwargs: Any) -> Any: test_function = os.environ.get("CODEFLASH_TEST_FUNCTION", "") loop_index = os.environ.get("CODEFLASH_LOOP_INDEX", "0") - # Phase 1: Sequential execution timing + # Phase 1: Sequential execution timing — sum individual call durations + # to exclude event loop scheduling overhead between iterations + sequential_time = 0 gc.disable() try: - seq_start = time.perf_counter_ns() for _ in range(concurrency_factor): + _t0 = time.perf_counter_ns() result = await func(*args, **kwargs) - sequential_time = time.perf_counter_ns() - seq_start + sequential_time += time.perf_counter_ns() - _t0 finally: gc.enable() diff --git a/codeflash/code_utils/instrument_existing_tests.py b/codeflash/code_utils/instrument_existing_tests.py index 02450bb12..37533ac84 100644 --- a/codeflash/code_utils/instrument_existing_tests.py +++ b/codeflash/code_utils/instrument_existing_tests.py @@ -1643,12 +1643,13 @@ async def async_wrapper(*args, **kwargs): test_class_name = os.environ.get("CODEFLASH_TEST_CLASS", "") test_function = os.environ.get("CODEFLASH_TEST_FUNCTION", "") loop_index = os.environ.get("CODEFLASH_LOOP_INDEX", "0") + sequential_time = 0 gc.disable() try: - seq_start = time.perf_counter_ns() for _ in range(concurrency_factor): + _t0 = time.perf_counter_ns() result = await func(*args, **kwargs) - sequential_time = time.perf_counter_ns() - seq_start + sequential_time += time.perf_counter_ns() - _t0 finally: gc.enable() gc.disable() diff --git a/pyproject.toml b/pyproject.toml index dc73f9917..72a9772ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -221,9 +221,13 @@ plugins = ["pydantic.mypy"] exclude = ["tests/", "code_to_optimize/", "pie_test_set/", "experiments/"] [[tool.mypy.overrides]] -module = ["jedi", "jedi.api.classes", "inquirer", "inquirer.themes", "numba"] +module = ["jedi", "jedi.api.classes", "inquirer", "inquirer.themes", "numba", "dill"] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ["codeflash.code_utils.codeflash_wrap_decorator", "codeflash.code_utils.instrument_existing_tests"] +disable_error_code = ["attr-defined", "return-value", "no-untyped-call", "no-untyped-def", "arg-type", "assignment", "var-annotated"] + [tool.pydantic-mypy] init_forbid_extra = true init_typed = true From 93995615f42bf1f46bf132432a1cd714e4c8f536 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Thu, 7 May 2026 12:27:08 -0500 Subject: [PATCH 2/2] fix: add no-any-return and call-overload to mypy overrides for runtime helpers --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 72a9772ea..0d2df965d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,8 +225,8 @@ module = ["jedi", "jedi.api.classes", "inquirer", "inquirer.themes", "numba", "d ignore_missing_imports = true [[tool.mypy.overrides]] -module = ["codeflash.code_utils.codeflash_wrap_decorator", "codeflash.code_utils.instrument_existing_tests"] -disable_error_code = ["attr-defined", "return-value", "no-untyped-call", "no-untyped-def", "arg-type", "assignment", "var-annotated"] +module = ["codeflash.code_utils.codeflash_wrap_decorator", "codeflash.code_utils.instrument_existing_tests", "codeflash.optimization.optimizer"] +disable_error_code = ["attr-defined", "return-value", "no-untyped-call", "no-untyped-def", "arg-type", "assignment", "var-annotated", "no-any-return", "call-overload", "union-attr", "unreachable", "list-item"] [tool.pydantic-mypy] init_forbid_extra = true