From 4602c38b61d2e7f99cd66981772be314d9278f79 Mon Sep 17 00:00:00 2001 From: Saurabh Misra Date: Fri, 18 Jul 2025 16:35:34 -0700 Subject: [PATCH 01/14] avoid logfire contamination Signed-off-by: Saurabh Misra --- codeflash/benchmarking/plugin/plugin.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index 6516fba38..c80f846f6 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -4,13 +4,24 @@ import sqlite3 import sys import time +from dataclasses import dataclass from pathlib import Path import pytest from codeflash.benchmarking.codeflash_trace import codeflash_trace from codeflash.code_utils.code_utils import module_name_from_file_path -from codeflash.models.models import BenchmarkKey + +# from codeflash.models.models import BenchmarkKey + + +@dataclass +class BenchmarkKey: + module_path: str + function_name: str + + def __str__(self) -> str: + return f"{self.module_path}::{self.function_name}" class CodeFlashBenchmarkPlugin: From 08c6a1a87e9e79cc4b7f67f442a4fb1dfc035bf4 Mon Sep 17 00:00:00 2001 From: Saurabh Misra Date: Mon, 21 Jul 2025 16:34:24 -0700 Subject: [PATCH 02/14] add some debugging lines Signed-off-by: Saurabh Misra --- codeflash/benchmarking/codeflash_trace.py | 4 ++++ codeflash/benchmarking/plugin/plugin.py | 9 +++++++++ codeflash/benchmarking/trace_benchmarks.py | 1 + 3 files changed, 14 insertions(+) diff --git a/codeflash/benchmarking/codeflash_trace.py b/codeflash/benchmarking/codeflash_trace.py index 249acdeb3..4923fd8f5 100644 --- a/codeflash/benchmarking/codeflash_trace.py +++ b/codeflash/benchmarking/codeflash_trace.py @@ -108,6 +108,7 @@ def __call__(self, func: Callable) -> Callable: @functools.wraps(func) def wrapper(*args, **kwargs) -> Any: # noqa: ANN002, ANN003, ANN401 + print("XXX CALLED THE CODEFLASH_TRACE MARKER") # Initialize thread-local active functions set if it doesn't exist if not hasattr(self._thread_local, "active_functions"): self._thread_local.active_functions = set() @@ -132,6 +133,9 @@ def wrapper(*args, **kwargs) -> Any: # noqa: ANN002, ANN003, ANN401 benchmark_function_name = os.environ.get("CODEFLASH_BENCHMARK_FUNCTION_NAME", "") benchmark_module_path = os.environ.get("CODEFLASH_BENCHMARK_MODULE_PATH", "") benchmark_line_number = os.environ.get("CODEFLASH_BENCHMARK_LINE_NUMBER", "") + print("XXX benchmark_function_name", benchmark_function_name) + print("XXX benchmark_module_path", benchmark_module_path) + print("XXX benchmark_line_number", benchmark_line_number) # Get class name class_name = "" qualname = func.__qualname__ diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index c80f846f6..166959701 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -55,6 +55,7 @@ def setup(self, trace_path: str, project_root: str) -> None: raise def write_benchmark_timings(self) -> None: + print("XXX ATTEMPTING TO WRITE THE BENCHMARK TIMINGS") if not self.benchmark_timings: return # No data to write @@ -97,6 +98,7 @@ def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[Benchmark """ # Initialize the result dictionary + print("XXX ATTEMPTING get_function_benchmark_timings") result = {} # Connect to the SQLite database @@ -156,6 +158,7 @@ def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: """ # Initialize the result dictionary + print("XXX ATTEMPTING get_benchmark_timings") result = {} overhead_by_benchmark = {} @@ -244,6 +247,7 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item marker = item.get_closest_marker("benchmark") if marker is not None: has_marker = True + print("XXX FOUND THE BENCHMARK MARKER") # Skip if neither fixture nor marker is present if not (has_fixture or has_marker): @@ -253,9 +257,11 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item class Benchmark: # noqa: D106 def __init__(self, request: pytest.FixtureRequest) -> None: self.request = request + print("XXX INITIALIZING THE BENCHMARK") def __call__(self, func, *args, **kwargs): # type: ignore # noqa: ANN001, ANN002, ANN003, ANN204, PGH003 """Handle both direct function calls and decorator usage.""" + print("XXX CALLED THE BENCHMARK") if args or kwargs: # Used as benchmark(func, *args, **kwargs) return self._run_benchmark(func, *args, **kwargs) @@ -269,6 +275,7 @@ def wrapped_func(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 """Actual benchmark implementation.""" + print("XXX RUNNING THE BENCHMARK!!") benchmark_module_path = module_name_from_file_path( Path(str(self.request.node.fspath)), Path(codeflash_benchmark_plugin.project_root) ) @@ -301,7 +308,9 @@ def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003 @pytest.fixture def benchmark(request: pytest.FixtureRequest) -> object: if not request.config.getoption("--codeflash-trace"): + print("XXX NOOOO PLS") return None + print("XXX BENCHMARK PLUGIN INITIATED") return CodeFlashBenchmarkPlugin.Benchmark(request) diff --git a/codeflash/benchmarking/trace_benchmarks.py b/codeflash/benchmarking/trace_benchmarks.py index e59b06656..32be26936 100644 --- a/codeflash/benchmarking/trace_benchmarks.py +++ b/codeflash/benchmarking/trace_benchmarks.py @@ -32,6 +32,7 @@ def trace_benchmarks_pytest( env=benchmark_env, timeout=timeout, ) + print(result.stdout) if result.returncode != 0: if "ERROR collecting" in result.stdout: # Pattern matches "===== ERRORS =====" (any number of =) and captures everything after From ea0cab7c6b561fc9c4e0fa4d5a2bfea0bfa75f88 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Mon, 21 Jul 2025 18:27:37 -0700 Subject: [PATCH 03/14] freeze the class --- codeflash/benchmarking/plugin/plugin.py | 24 +++++-------------- .../pytest_new_process_trace_benchmarks.py | 2 +- codeflash/optimization/optimizer.py | 14 +++++------ 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index 166959701..31d2eb4b7 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -12,10 +12,8 @@ from codeflash.benchmarking.codeflash_trace import codeflash_trace from codeflash.code_utils.code_utils import module_name_from_file_path -# from codeflash.models.models import BenchmarkKey - -@dataclass +@dataclass(frozen=True) class BenchmarkKey: module_path: str function_name: str @@ -192,6 +190,7 @@ def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: # Create the benchmark key (file::function::line) benchmark_key = BenchmarkKey(module_path=benchmark_file, function_name=benchmark_func) + print(f"XXX Processing benchmark: {benchmark_key}") # Subtract overhead from total time overhead = overhead_by_benchmark.get(benchmark_key, 0) result[benchmark_key] = time_ns - overhead @@ -232,26 +231,16 @@ def pytest_configure(config: pytest.Config) -> None: @staticmethod def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - # Skip tests that don't have the benchmark fixture + # Skip tests that don't have the benchmark marker if not config.getoption("--codeflash-trace"): return - skip_no_benchmark = pytest.mark.skip(reason="Test requires benchmark fixture") for item in items: - # Check for direct benchmark fixture usage - has_fixture = hasattr(item, "fixturenames") and "benchmark" in item.fixturenames - # Check for @pytest.mark.benchmark marker - has_marker = False if hasattr(item, "get_closest_marker"): marker = item.get_closest_marker("benchmark") - if marker is not None: - has_marker = True - print("XXX FOUND THE BENCHMARK MARKER") - - # Skip if neither fixture nor marker is present - if not (has_fixture or has_marker): - item.add_marker(skip_no_benchmark) + if marker is None: + item.add_marker(pytest.mark.skip(reason="Test requires benchmark marker")) # Benchmark fixture class Benchmark: # noqa: D106 @@ -307,11 +296,10 @@ def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003 @staticmethod @pytest.fixture def benchmark(request: pytest.FixtureRequest) -> object: + """Fixture to provide the benchmark functionality.""" if not request.config.getoption("--codeflash-trace"): - print("XXX NOOOO PLS") return None print("XXX BENCHMARK PLUGIN INITIATED") - return CodeFlashBenchmarkPlugin.Benchmark(request) diff --git a/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py b/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py index f8650d9fb..1148539a0 100644 --- a/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py +++ b/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py @@ -28,12 +28,12 @@ "-p", "no:profiling", "-s", + "-vv", "-o", "addopts=", ], plugins=[codeflash_benchmark_plugin], ) # Errors will be printed to stdout, not stderr - except Exception as e: print(f"Failed to collect tests: {e!s}", file=sys.stderr) exitcode = -1 diff --git a/codeflash/optimization/optimizer.py b/codeflash/optimization/optimizer.py index 63a32199e..710a70a5b 100644 --- a/codeflash/optimization/optimizer.py +++ b/codeflash/optimization/optimizer.py @@ -89,13 +89,13 @@ def run_benchmarks( logger.info( f"No valid benchmarks found in {self.args.benchmarks_root} for functions to optimize, continuing optimization" ) - else: - function_benchmark_timings = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(trace_file) - total_benchmark_timings = CodeFlashBenchmarkPlugin.get_benchmark_timings(trace_file) - function_to_results = validate_and_format_benchmark_table( - function_benchmark_timings, total_benchmark_timings - ) - print_benchmark_table(function_to_results) + raise SystemExit # noqa: TRY301 + function_benchmark_timings = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(trace_file) + total_benchmark_timings = CodeFlashBenchmarkPlugin.get_benchmark_timings(trace_file) + function_to_results = validate_and_format_benchmark_table( + function_benchmark_timings, total_benchmark_timings + ) + print_benchmark_table(function_to_results) except Exception as e: logger.info(f"Error while tracing existing benchmarks: {e}") logger.info("Information on existing benchmarks will not be available for this run.") From 9ddd76358bd6c3a2997873cf95310b0a843b949b Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 12:18:41 -0700 Subject: [PATCH 04/14] module_name traverse up and cleanup prints --- codeflash/benchmarking/plugin/plugin.py | 11 ++--------- codeflash/code_utils/code_utils.py | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index 31d2eb4b7..d69939f6e 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -53,7 +53,6 @@ def setup(self, trace_path: str, project_root: str) -> None: raise def write_benchmark_timings(self) -> None: - print("XXX ATTEMPTING TO WRITE THE BENCHMARK TIMINGS") if not self.benchmark_timings: return # No data to write @@ -96,7 +95,6 @@ def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[Benchmark """ # Initialize the result dictionary - print("XXX ATTEMPTING get_function_benchmark_timings") result = {} # Connect to the SQLite database @@ -156,7 +154,6 @@ def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: """ # Initialize the result dictionary - print("XXX ATTEMPTING get_benchmark_timings") result = {} overhead_by_benchmark = {} @@ -190,7 +187,6 @@ def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: # Create the benchmark key (file::function::line) benchmark_key = BenchmarkKey(module_path=benchmark_file, function_name=benchmark_func) - print(f"XXX Processing benchmark: {benchmark_key}") # Subtract overhead from total time overhead = overhead_by_benchmark.get(benchmark_key, 0) result[benchmark_key] = time_ns - overhead @@ -246,11 +242,9 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item class Benchmark: # noqa: D106 def __init__(self, request: pytest.FixtureRequest) -> None: self.request = request - print("XXX INITIALIZING THE BENCHMARK") def __call__(self, func, *args, **kwargs): # type: ignore # noqa: ANN001, ANN002, ANN003, ANN204, PGH003 """Handle both direct function calls and decorator usage.""" - print("XXX CALLED THE BENCHMARK") if args or kwargs: # Used as benchmark(func, *args, **kwargs) return self._run_benchmark(func, *args, **kwargs) @@ -264,10 +258,10 @@ def wrapped_func(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 """Actual benchmark implementation.""" - print("XXX RUNNING THE BENCHMARK!!") benchmark_module_path = module_name_from_file_path( - Path(str(self.request.node.fspath)), Path(codeflash_benchmark_plugin.project_root) + Path(str(self.request.node.fspath)), Path(codeflash_benchmark_plugin.project_root), traverse_up=True ) + benchmark_function_name = self.request.node.name line_number = int(str(sys._getframe(2).f_lineno)) # 2 frames up in the call stack # noqa: SLF001 # Set env vars @@ -299,7 +293,6 @@ def benchmark(request: pytest.FixtureRequest) -> object: """Fixture to provide the benchmark functionality.""" if not request.config.getoption("--codeflash-trace"): return None - print("XXX BENCHMARK PLUGIN INITIATED") return CodeFlashBenchmarkPlugin.Benchmark(request) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 82a5b9791..bd235aedd 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -109,9 +109,21 @@ def get_qualified_name(module_name: str, full_qualified_name: str) -> str: return full_qualified_name[len(module_name) + 1 :] -def module_name_from_file_path(file_path: Path, project_root_path: Path) -> str: - relative_path = file_path.relative_to(project_root_path) - return relative_path.with_suffix("").as_posix().replace("/", ".") +def module_name_from_file_path(file_path: Path, project_root_path: Path, *, traverse_up: bool = True) -> str: + try: + relative_path = file_path.relative_to(project_root_path) + return relative_path.with_suffix("").as_posix().replace("/", ".") + except ValueError: + if traverse_up: + parent = file_path.parent + while parent not in (project_root_path, parent.parent): + try: + relative_path = file_path.relative_to(parent) + return relative_path.with_suffix("").as_posix().replace("/", ".") + except ValueError: + parent = parent.parent + msg = f"File {file_path} is not within the project root {project_root_path}." + raise ValueError(msg) # noqa: B904 def file_path_from_module_name(module_name: str, project_root_path: Path) -> Path: From 72bd8e73d292b729edd2477d6e099f8e94a277c4 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 12:26:50 -0700 Subject: [PATCH 05/14] remove debug print --- codeflash/benchmarking/pytest_new_process_trace_benchmarks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py b/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py index 1148539a0..b32922533 100644 --- a/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py +++ b/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py @@ -28,7 +28,6 @@ "-p", "no:profiling", "-s", - "-vv", "-o", "addopts=", ], From 2ad963b6102fc7a7e04a02d28d471a660fa4bb0b Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 13:31:52 -0700 Subject: [PATCH 06/14] add marker so that test doesn't crash --- codeflash/benchmarking/pytest_new_process_trace_benchmarks.py | 1 + codeflash/benchmarking/trace_benchmarks.py | 1 - codeflash/discovery/pytest_new_process_discovery.py | 3 +++ codeflash/optimization/optimizer.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py b/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py index b32922533..f8650d9fb 100644 --- a/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py +++ b/codeflash/benchmarking/pytest_new_process_trace_benchmarks.py @@ -33,6 +33,7 @@ ], plugins=[codeflash_benchmark_plugin], ) # Errors will be printed to stdout, not stderr + except Exception as e: print(f"Failed to collect tests: {e!s}", file=sys.stderr) exitcode = -1 diff --git a/codeflash/benchmarking/trace_benchmarks.py b/codeflash/benchmarking/trace_benchmarks.py index 32be26936..e59b06656 100644 --- a/codeflash/benchmarking/trace_benchmarks.py +++ b/codeflash/benchmarking/trace_benchmarks.py @@ -32,7 +32,6 @@ def trace_benchmarks_pytest( env=benchmark_env, timeout=timeout, ) - print(result.stdout) if result.returncode != 0: if "ERROR collecting" in result.stdout: # Pattern matches "===== ERRORS =====" (any number of =) and captures everything after diff --git a/codeflash/discovery/pytest_new_process_discovery.py b/codeflash/discovery/pytest_new_process_discovery.py index 9d695b52a..41b764f7d 100644 --- a/codeflash/discovery/pytest_new_process_discovery.py +++ b/codeflash/discovery/pytest_new_process_discovery.py @@ -23,6 +23,9 @@ def pytest_collection_modifyitems(self, items) -> None: if "benchmark" in item.fixturenames: item.add_marker(skip_benchmark) + def pytest_configure(self, config) -> None: + config.addinivalue_line("markers", "benchmark: placeholder") + def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, str]]: test_results = [] diff --git a/codeflash/optimization/optimizer.py b/codeflash/optimization/optimizer.py index 710a70a5b..278e00e77 100644 --- a/codeflash/optimization/optimizer.py +++ b/codeflash/optimization/optimizer.py @@ -64,6 +64,7 @@ def run_benchmarks( from codeflash.benchmarking.trace_benchmarks import trace_benchmarks_pytest from codeflash.benchmarking.utils import print_benchmark_table, validate_and_format_benchmark_table + console.rule() with progress_bar( f"Running benchmarks in {self.args.benchmarks_root}", transient=True, revert_to_print=bool(get_pr_number()) ): From 32d991973560b698c59506f0452cf720a0c931ea Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 14:11:14 -0700 Subject: [PATCH 07/14] add the plugin as a pytest entry point --- codeflash/benchmarking/plugin/plugin.py | 33 +++++++++++++++++++------ codeflash/optimization/optimizer.py | 14 +++++------ pyproject.toml | 4 ++- uv.lock | 30 ++++------------------ 4 files changed, 41 insertions(+), 40 deletions(-) diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index d69939f6e..a120a9496 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib.util import os import sqlite3 import sys @@ -12,6 +13,8 @@ from codeflash.benchmarking.codeflash_trace import codeflash_trace from codeflash.code_utils.code_utils import module_name_from_file_path +IS_PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None + @dataclass(frozen=True) class BenchmarkKey: @@ -212,19 +215,18 @@ def pytest_sessionfinish(self, session, exitstatus) -> None: # noqa: ANN001, AR def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption("--codeflash-trace", action="store_true", default=False, help="Enable CodeFlash tracing") - @staticmethod - def pytest_plugin_registered(plugin, manager) -> None: # noqa: ANN001 - # Not necessary since run with -p no:benchmark, but just in case - if hasattr(plugin, "name") and plugin.name == "pytest-benchmark": - manager.unregister(plugin) - @staticmethod def pytest_configure(config: pytest.Config) -> None: - """Register the benchmark marker.""" + """Register the benchmark marker and disable conflicting plugins.""" config.addinivalue_line( "markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing" ) + if config.getoption("--codeflash-trace") and IS_PYTEST_BENCHMARK_INSTALLED: + object.__setattr__(config.option, "benchmark_disable", True) + config.pluginmanager.set_blocked("pytest_benchmark") + config.pluginmanager.set_blocked("pytest-benchmark") + @staticmethod def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: # Skip tests that don't have the benchmark marker @@ -297,3 +299,20 @@ def benchmark(request: pytest.FixtureRequest) -> object: codeflash_benchmark_plugin = CodeFlashBenchmarkPlugin() + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing") + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--codeflash-trace", action="store_true", default=False, help="Enable CodeFlash tracing for benchmarks" + ) + + +@pytest.fixture +def benchmark(request: pytest.FixtureRequest) -> object: + if not request.config.getoption("--codeflash-trace"): + return lambda func, *args, **kwargs: func(*args, **kwargs) + return codeflash_benchmark_plugin.Benchmark(request) diff --git a/codeflash/optimization/optimizer.py b/codeflash/optimization/optimizer.py index 278e00e77..5999c5fa1 100644 --- a/codeflash/optimization/optimizer.py +++ b/codeflash/optimization/optimizer.py @@ -90,13 +90,13 @@ def run_benchmarks( logger.info( f"No valid benchmarks found in {self.args.benchmarks_root} for functions to optimize, continuing optimization" ) - raise SystemExit # noqa: TRY301 - function_benchmark_timings = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(trace_file) - total_benchmark_timings = CodeFlashBenchmarkPlugin.get_benchmark_timings(trace_file) - function_to_results = validate_and_format_benchmark_table( - function_benchmark_timings, total_benchmark_timings - ) - print_benchmark_table(function_to_results) + else: + function_benchmark_timings = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(trace_file) + total_benchmark_timings = CodeFlashBenchmarkPlugin.get_benchmark_timings(trace_file) + function_to_results = validate_and_format_benchmark_table( + function_benchmark_timings, total_benchmark_timings + ) + print_benchmark_table(function_to_results) except Exception as e: logger.info(f"Error while tracing existing benchmarks: {e}") logger.info("Information on existing benchmarks will not be available for this run.") diff --git a/pyproject.toml b/pyproject.toml index 1862bed91..3dba0e759 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,6 @@ dev = [ "types-openpyxl>=3.1.5.20241020", "types-regex>=2024.9.11.20240912", "types-python-dateutil>=2.9.0.20241003", - "pytest-benchmark>=5.1.0", "types-gevent>=24.11.0.20241230,<25", "types-greenlet>=3.1.0.20241221,<4", "types-pexpect>=4.9.0.20241208,<5", @@ -300,3 +299,6 @@ markers = [ [build-system] requires = ["hatchling", "uv-dynamic-versioning"] build-backend = "hatchling.build" + +[project.entry-points.pytest11] +codeflash = "codeflash.benchmarking.plugin.plugin" diff --git a/uv.lock b/uv.lock index afb3f2332..d8a2b77c6 100644 --- a/uv.lock +++ b/uv.lock @@ -247,7 +247,6 @@ dev = [ { name = "pandas-stubs", version = "2.2.2.240807", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pandas-stubs", version = "2.2.2.240909", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pre-commit" }, - { name = "pytest-benchmark" }, { name = "ruff" }, { name = "types-cffi" }, { name = "types-colorama" }, @@ -303,7 +302,6 @@ dev = [ { name = "mypy", specifier = ">=1.13" }, { name = "pandas-stubs", specifier = ">=2.2.2.240807,<2.2.3.241009" }, { name = "pre-commit", specifier = ">=4.2.0,<5" }, - { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "ruff", specifier = ">=0.7.0" }, { name = "types-cffi", specifier = ">=1.16.0.20240331" }, { name = "types-colorama", specifier = ">=0.4.15.20240311" }, @@ -511,7 +509,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -937,10 +935,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/55/2cb24ea48aa30c99f805921c1c7860c1f45c0e811e44ee4e6a155668de06/lxml-6.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563", size = 4952289, upload-time = "2025-06-28T18:47:25.602Z" }, { url = "https://files.pythonhosted.org/packages/31/c0/b25d9528df296b9a3306ba21ff982fc5b698c45ab78b94d18c2d6ae71fd9/lxml-6.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7", size = 5111310, upload-time = "2025-06-28T18:47:28.136Z" }, { url = "https://files.pythonhosted.org/packages/e9/af/681a8b3e4f668bea6e6514cbcb297beb6de2b641e70f09d3d78655f4f44c/lxml-6.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7", size = 5025457, upload-time = "2025-06-26T16:26:15.068Z" }, + { url = "https://files.pythonhosted.org/packages/99/b6/3a7971aa05b7be7dfebc7ab57262ec527775c2c3c5b2f43675cac0458cad/lxml-6.0.0-cp312-cp312-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991", size = 5657016, upload-time = "2025-07-03T19:19:06.008Z" }, { url = "https://files.pythonhosted.org/packages/69/f8/693b1a10a891197143c0673fcce5b75fc69132afa81a36e4568c12c8faba/lxml-6.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da", size = 5257565, upload-time = "2025-06-26T16:26:17.906Z" }, { url = "https://files.pythonhosted.org/packages/a8/96/e08ff98f2c6426c98c8964513c5dab8d6eb81dadcd0af6f0c538ada78d33/lxml-6.0.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e", size = 4713390, upload-time = "2025-06-26T16:26:20.292Z" }, { url = "https://files.pythonhosted.org/packages/a8/83/6184aba6cc94d7413959f6f8f54807dc318fdcd4985c347fe3ea6937f772/lxml-6.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741", size = 5066103, upload-time = "2025-06-26T16:26:22.765Z" }, { url = "https://files.pythonhosted.org/packages/ee/01/8bf1f4035852d0ff2e36a4d9aacdbcc57e93a6cd35a54e05fa984cdf73ab/lxml-6.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3", size = 4791428, upload-time = "2025-06-26T16:26:26.461Z" }, + { url = "https://files.pythonhosted.org/packages/29/31/c0267d03b16954a85ed6b065116b621d37f559553d9339c7dcc4943a76f1/lxml-6.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16", size = 5678523, upload-time = "2025-07-03T19:19:09.837Z" }, { url = "https://files.pythonhosted.org/packages/5c/f7/5495829a864bc5f8b0798d2b52a807c89966523140f3d6fa3a58ab6720ea/lxml-6.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0", size = 5281290, upload-time = "2025-06-26T16:26:29.406Z" }, { url = "https://files.pythonhosted.org/packages/79/56/6b8edb79d9ed294ccc4e881f4db1023af56ba451909b9ce79f2a2cd7c532/lxml-6.0.0-cp312-cp312-win32.whl", hash = "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a", size = 3613495, upload-time = "2025-06-26T16:26:31.588Z" }, { url = "https://files.pythonhosted.org/packages/0b/1e/cc32034b40ad6af80b6fd9b66301fc0f180f300002e5c3eb5a6110a93317/lxml-6.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3", size = 4014711, upload-time = "2025-06-26T16:26:33.723Z" }, @@ -951,10 +951,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/46/3572761efc1bd45fcafb44a63b3b0feeb5b3f0066886821e94b0254f9253/lxml-6.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81", size = 4947559, upload-time = "2025-06-28T18:47:31.091Z" }, { url = "https://files.pythonhosted.org/packages/94/8a/5e40de920e67c4f2eef9151097deb9b52d86c95762d8ee238134aff2125d/lxml-6.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1", size = 5102143, upload-time = "2025-06-28T18:47:33.612Z" }, { url = "https://files.pythonhosted.org/packages/7c/4b/20555bdd75d57945bdabfbc45fdb1a36a1a0ff9eae4653e951b2b79c9209/lxml-6.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24", size = 5021931, upload-time = "2025-06-26T16:26:47.503Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/cf03b412f3763d4ca23b25e70c96a74cfece64cec3addf1c4ec639586b13/lxml-6.0.0-cp313-cp313-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a", size = 5645469, upload-time = "2025-07-03T19:19:13.32Z" }, { url = "https://files.pythonhosted.org/packages/d4/dd/39c8507c16db6031f8c1ddf70ed95dbb0a6d466a40002a3522c128aba472/lxml-6.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29", size = 5247467, upload-time = "2025-06-26T16:26:49.998Z" }, { url = "https://files.pythonhosted.org/packages/4d/56/732d49def0631ad633844cfb2664563c830173a98d5efd9b172e89a4800d/lxml-6.0.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4", size = 4720601, upload-time = "2025-06-26T16:26:52.564Z" }, { url = "https://files.pythonhosted.org/packages/8f/7f/6b956fab95fa73462bca25d1ea7fc8274ddf68fb8e60b78d56c03b65278e/lxml-6.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca", size = 5060227, upload-time = "2025-06-26T16:26:55.054Z" }, { url = "https://files.pythonhosted.org/packages/97/06/e851ac2924447e8b15a294855caf3d543424364a143c001014d22c8ca94c/lxml-6.0.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf", size = 4790637, upload-time = "2025-06-26T16:26:57.384Z" }, + { url = "https://files.pythonhosted.org/packages/06/d4/fd216f3cd6625022c25b336c7570d11f4a43adbaf0a56106d3d496f727a7/lxml-6.0.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f", size = 5662049, upload-time = "2025-07-03T19:19:16.409Z" }, { url = "https://files.pythonhosted.org/packages/52/03/0e764ce00b95e008d76b99d432f1807f3574fb2945b496a17807a1645dbd/lxml-6.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef", size = 5272430, upload-time = "2025-06-26T16:27:00.031Z" }, { url = "https://files.pythonhosted.org/packages/5f/01/d48cc141bc47bc1644d20fe97bbd5e8afb30415ec94f146f2f76d0d9d098/lxml-6.0.0-cp313-cp313-win32.whl", hash = "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181", size = 3612896, upload-time = "2025-06-26T16:27:04.251Z" }, { url = "https://files.pythonhosted.org/packages/f4/87/6456b9541d186ee7d4cb53bf1b9a0d7f3b1068532676940fdd594ac90865/lxml-6.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e", size = 4013132, upload-time = "2025-06-26T16:27:06.415Z" }, @@ -1438,15 +1440,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - [[package]] name = "pydantic" version = "2.11.7" @@ -1611,19 +1604,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, ] -[[package]] -name = "pytest-benchmark" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py-cpuinfo" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" }, -] - [[package]] name = "pytest-timeout" version = "2.4.0" From c6776dc06b532b1a83f9db844608bb92825b4dd2 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 17:39:50 -0700 Subject: [PATCH 08/14] cleanup plugin --- codeflash/benchmarking/plugin/plugin.py | 45 ++++++++++--------------- tests/test_trace_benchmarks.py | 2 +- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index a120a9496..f7da4b1c6 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -211,34 +211,27 @@ def pytest_sessionfinish(self, session, exitstatus) -> None: # noqa: ANN001, AR # Close the database connection self.close() - @staticmethod - def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption("--codeflash-trace", action="store_true", default=False, help="Enable CodeFlash tracing") - - @staticmethod - def pytest_configure(config: pytest.Config) -> None: - """Register the benchmark marker and disable conflicting plugins.""" - config.addinivalue_line( - "markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing" - ) - - if config.getoption("--codeflash-trace") and IS_PYTEST_BENCHMARK_INSTALLED: - object.__setattr__(config.option, "benchmark_disable", True) - config.pluginmanager.set_blocked("pytest_benchmark") - config.pluginmanager.set_blocked("pytest-benchmark") - @staticmethod def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - # Skip tests that don't have the benchmark marker + # Skip tests that don't have the benchmark fixture if not config.getoption("--codeflash-trace"): return + skip_no_benchmark = pytest.mark.skip(reason="Test requires benchmark fixture") for item in items: + # Check for direct benchmark fixture usage + has_fixture = hasattr(item, "fixturenames") and "benchmark" in item.fixturenames + # Check for @pytest.mark.benchmark marker + has_marker = False if hasattr(item, "get_closest_marker"): marker = item.get_closest_marker("benchmark") - if marker is None: - item.add_marker(pytest.mark.skip(reason="Test requires benchmark marker")) + if marker is not None: + has_marker = True + + # Skip if neither fixture nor marker is present + if not (has_fixture or has_marker): + item.add_marker(skip_no_benchmark) # Benchmark fixture class Benchmark: # noqa: D106 @@ -289,21 +282,19 @@ def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003 return result - @staticmethod - @pytest.fixture - def benchmark(request: pytest.FixtureRequest) -> object: - """Fixture to provide the benchmark functionality.""" - if not request.config.getoption("--codeflash-trace"): - return None - return CodeFlashBenchmarkPlugin.Benchmark(request) - codeflash_benchmark_plugin = CodeFlashBenchmarkPlugin() def pytest_configure(config: pytest.Config) -> None: + """Register the benchmark marker and disable conflicting plugins.""" config.addinivalue_line("markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing") + if config.getoption("--codeflash-trace") and IS_PYTEST_BENCHMARK_INSTALLED: + config.option.benchmark_disable = True + config.pluginmanager.set_blocked("pytest_benchmark") + config.pluginmanager.set_blocked("pytest-benchmark") + def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( diff --git a/tests/test_trace_benchmarks.py b/tests/test_trace_benchmarks.py index 2d5a3c6e0..adc00847b 100644 --- a/tests/test_trace_benchmarks.py +++ b/tests/test_trace_benchmarks.py @@ -33,7 +33,7 @@ def test_trace_benchmarks() -> None: function_calls = cursor.fetchall() # Assert the length of function calls - assert len(function_calls) == 8, f"Expected 8 function calls, but got {len(function_calls)}" + assert len(function_calls) == 7, f"Expected 7 function calls, but got {len(function_calls)}" bubble_sort_path = (project_root / "bubble_sort_codeflash_trace.py").as_posix() process_and_bubble_sort_path = (project_root / "process_and_bubble_sort_codeflash_trace.py").as_posix() From 609144f1ff3752f22b80609fb22a87812ecc4c3b Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 17:46:37 -0700 Subject: [PATCH 09/14] Update unit-tests.yaml --- .github/workflows/unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 4a04b105d..d27ff8eee 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -30,4 +30,4 @@ jobs: run: uv sync - name: Unit tests - run: uv run pytest tests/ --benchmark-skip -m "not ci_skip" \ No newline at end of file + run: uv run pytest tests/ \ No newline at end of file From 7a6d394ad6036935f8932494863f63fea5a8d92b Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 17:51:58 -0700 Subject: [PATCH 10/14] not used --- codeflash/discovery/pytest_new_process_discovery.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/codeflash/discovery/pytest_new_process_discovery.py b/codeflash/discovery/pytest_new_process_discovery.py index 41b764f7d..9d695b52a 100644 --- a/codeflash/discovery/pytest_new_process_discovery.py +++ b/codeflash/discovery/pytest_new_process_discovery.py @@ -23,9 +23,6 @@ def pytest_collection_modifyitems(self, items) -> None: if "benchmark" in item.fixturenames: item.add_marker(skip_benchmark) - def pytest_configure(self, config) -> None: - config.addinivalue_line("markers", "benchmark: placeholder") - def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, str]]: test_results = [] From 1296da23e0f7e17fe2538c870d41dd3468d56552 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 18:09:50 -0700 Subject: [PATCH 11/14] revert debug prints --- codeflash/benchmarking/codeflash_trace.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/codeflash/benchmarking/codeflash_trace.py b/codeflash/benchmarking/codeflash_trace.py index 4923fd8f5..249acdeb3 100644 --- a/codeflash/benchmarking/codeflash_trace.py +++ b/codeflash/benchmarking/codeflash_trace.py @@ -108,7 +108,6 @@ def __call__(self, func: Callable) -> Callable: @functools.wraps(func) def wrapper(*args, **kwargs) -> Any: # noqa: ANN002, ANN003, ANN401 - print("XXX CALLED THE CODEFLASH_TRACE MARKER") # Initialize thread-local active functions set if it doesn't exist if not hasattr(self._thread_local, "active_functions"): self._thread_local.active_functions = set() @@ -133,9 +132,6 @@ def wrapper(*args, **kwargs) -> Any: # noqa: ANN002, ANN003, ANN401 benchmark_function_name = os.environ.get("CODEFLASH_BENCHMARK_FUNCTION_NAME", "") benchmark_module_path = os.environ.get("CODEFLASH_BENCHMARK_MODULE_PATH", "") benchmark_line_number = os.environ.get("CODEFLASH_BENCHMARK_LINE_NUMBER", "") - print("XXX benchmark_function_name", benchmark_function_name) - print("XXX benchmark_module_path", benchmark_module_path) - print("XXX benchmark_line_number", benchmark_line_number) # Get class name class_name = "" qualname = func.__qualname__ From 3159233861f34a28ffd49b93791a0ecf6defcea5 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 18:24:52 -0700 Subject: [PATCH 12/14] fix test failures --- codeflash/benchmarking/plugin/plugin.py | 18 ++++++++---------- codeflash/models/models.py | 3 ++- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index f7da4b1c6..f6fba65af 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -5,24 +5,18 @@ import sqlite3 import sys import time -from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING import pytest from codeflash.benchmarking.codeflash_trace import codeflash_trace from codeflash.code_utils.code_utils import module_name_from_file_path -IS_PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None - - -@dataclass(frozen=True) -class BenchmarkKey: - module_path: str - function_name: str +if TYPE_CHECKING: + from codeflash.models.models import BenchmarkKey - def __str__(self) -> str: - return f"{self.module_path}::{self.function_name}" +IS_PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None class CodeFlashBenchmarkPlugin: @@ -83,6 +77,8 @@ def close(self) -> None: @staticmethod def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[BenchmarkKey, int]]: + from codeflash.models.models import BenchmarkKey + """Process the trace file and extract timing data for all functions. Args: @@ -143,6 +139,8 @@ def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[Benchmark @staticmethod def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: + from codeflash.models.models import BenchmarkKey + """Extract total benchmark timings from trace files. Args: diff --git a/codeflash/models/models.py b/codeflash/models/models.py index e96d12423..f740e4dab 100644 --- a/codeflash/models/models.py +++ b/codeflash/models/models.py @@ -13,6 +13,7 @@ import re import sys from collections.abc import Collection +from dataclasses import dataclass as dcdataclass from enum import Enum, IntEnum from pathlib import Path from re import Pattern @@ -83,7 +84,7 @@ class BestOptimization(BaseModel): winning_replay_benchmarking_test_results: Optional[TestResults] = None -@dataclass(frozen=True) +@dcdataclass(frozen=True) class BenchmarkKey: module_path: str function_name: str From bc44a2fd14c186070d6e8d22232c8bb6dead9e7e Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Wed, 23 Jul 2025 19:19:29 -0700 Subject: [PATCH 13/14] Update models.py --- codeflash/models/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codeflash/models/models.py b/codeflash/models/models.py index f740e4dab..e96d12423 100644 --- a/codeflash/models/models.py +++ b/codeflash/models/models.py @@ -13,7 +13,6 @@ import re import sys from collections.abc import Collection -from dataclasses import dataclass as dcdataclass from enum import Enum, IntEnum from pathlib import Path from re import Pattern @@ -84,7 +83,7 @@ class BestOptimization(BaseModel): winning_replay_benchmarking_test_results: Optional[TestResults] = None -@dcdataclass(frozen=True) +@dataclass(frozen=True) class BenchmarkKey: module_path: str function_name: str From 79d6e4c419b26f0dcde6b1fec93d7fb140a3a6ca Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Thu, 24 Jul 2025 13:37:05 -0700 Subject: [PATCH 14/14] Update code_utils.py --- codeflash/code_utils/code_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index bd235aedd..59c75a00b 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -109,7 +109,7 @@ def get_qualified_name(module_name: str, full_qualified_name: str) -> str: return full_qualified_name[len(module_name) + 1 :] -def module_name_from_file_path(file_path: Path, project_root_path: Path, *, traverse_up: bool = True) -> str: +def module_name_from_file_path(file_path: Path, project_root_path: Path, *, traverse_up: bool = False) -> str: try: relative_path = file_path.relative_to(project_root_path) return relative_path.with_suffix("").as_posix().replace("/", ".")