From c002c24e2214cce0a43f56cd5987c7b91fc2317b Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 13:41:09 -0700 Subject: [PATCH 1/8] maybe something other than logger info --- codeflash/optimization/function_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index fe4357839..175075b11 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -1186,7 +1186,7 @@ def run_and_parse_tests( ) return TestResults(), None if run_result.returncode != 0 and testing_type == TestingMode.BEHAVIOR: - logger.debug( + logger.info( f"Nonzero return code {run_result.returncode} when running tests in " f"{', '.join([str(f.instrumented_behavior_file_path) for f in test_files.test_files])}.\n" f"stdout: {run_result.stdout}\n" From 5d82d8694c2f20ece1c44746f726182bc96d231e Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 14:14:06 -0700 Subject: [PATCH 2/8] rich text output for user to look at --- codeflash/optimization/function_optimizer.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 175075b11..8cc85093c 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re + import ast import concurrent.futures import os @@ -1186,12 +1188,20 @@ def run_and_parse_tests( ) return TestResults(), None if run_result.returncode != 0 and testing_type == TestingMode.BEHAVIOR: - logger.info( + logger.debug( f"Nonzero return code {run_result.returncode} when running tests in " f"{', '.join([str(f.instrumented_behavior_file_path) for f in test_files.test_files])}.\n" f"stdout: {run_result.stdout}\n" f"stderr: {run_result.stderr}\n" ) + if 'ModuleNotFoundError' in run_result.stdout: + from rich.text import Text + match = re.search(r'^.*ModuleNotFoundError.*$', run_result.stdout, re.MULTILINE).group() + panel = Panel( + Text.from_markup(f"⚠️ {match} ", style="bold red"), + expand=False, + ) + console.print(panel) if testing_type in {TestingMode.BEHAVIOR, TestingMode.PERFORMANCE}: results, coverage_results = parse_test_results( test_xml_path=result_file_path, From 1c6a11c1a5cfb5279f8ac0699ba094d901e62985 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 14:45:51 -0700 Subject: [PATCH 3/8] precommit fix --- codeflash/optimization/function_optimizer.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 8cc85093c..0c9edfdee 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -1,10 +1,9 @@ from __future__ import annotations -import re - import ast import concurrent.futures import os +import re import subprocess import time import uuid @@ -1194,13 +1193,11 @@ def run_and_parse_tests( f"stdout: {run_result.stdout}\n" f"stderr: {run_result.stderr}\n" ) - if 'ModuleNotFoundError' in run_result.stdout: + if "ModuleNotFoundError" in run_result.stdout: from rich.text import Text - match = re.search(r'^.*ModuleNotFoundError.*$', run_result.stdout, re.MULTILINE).group() - panel = Panel( - Text.from_markup(f"⚠️ {match} ", style="bold red"), - expand=False, - ) + + match = re.search(r"^.*ModuleNotFoundError.*$", run_result.stdout, re.MULTILINE).group() + panel = Panel(Text.from_markup(f"⚠️ {match} ", style="bold red"), expand=False) console.print(panel) if testing_type in {TestingMode.BEHAVIOR, TestingMode.PERFORMANCE}: results, coverage_results = parse_test_results( From c2d494f6bba73e22b7a2259af401fb5fd43bc507 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 14:58:35 -0700 Subject: [PATCH 4/8] test --- tests/test_test_runner.py | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/test_test_runner.py b/tests/test_test_runner.py index 2c67a644c..ef7e69968 100644 --- a/tests/test_test_runner.py +++ b/tests/test_test_runner.py @@ -1,3 +1,5 @@ +import re + import os import tempfile from pathlib import Path @@ -96,3 +98,51 @@ def test_sort(): ) assert results[0].did_pass, "Test did not pass as expected" result_file.unlink(missing_ok=True) + + code = """import torch +def sorter(arr): + print(torch.ones(1)) + arr.sort() + return arr + +def test_sort(): + arr = [5, 4, 3, 2, 1, 0] + output = sorter(arr) + assert output == [0, 1, 2, 3, 4, 5] + """ + cur_dir_path = Path(__file__).resolve().parent + config = TestConfig( + tests_root=cur_dir_path, + project_root_path=cur_dir_path, + test_framework="pytest", + tests_project_rootdir=cur_dir_path.parent, + ) + + test_env = os.environ.copy() + test_env["CODEFLASH_TEST_ITERATION"] = "0" + test_env["CODEFLASH_TRACER_DISABLE"] = "1" + if "PYTHONPATH" not in test_env: + test_env["PYTHONPATH"] = str(config.project_root_path) + else: + test_env["PYTHONPATH"] += os.pathsep + str(config.project_root_path) + + with tempfile.NamedTemporaryFile(prefix="test_xx", suffix=".py", dir=cur_dir_path) as fp: + test_files = TestFiles( + test_files=[TestFile(instrumented_behavior_file_path=Path(fp.name), test_type=TestType.EXISTING_UNIT_TEST)] + ) + fp.write(code.encode("utf-8")) + fp.flush() + result_file, process, _, _ = run_behavioral_tests( + test_files, + test_framework=config.test_framework, + cwd=Path(config.project_root_path), + test_env=test_env, + pytest_timeout=1, + pytest_target_runtime_seconds=1, + ) + results = parse_test_xml( + test_xml_file_path=result_file, test_files=test_files, test_config=config, run_result=process + ) + match = re.search(r"^.*ModuleNotFoundError.*$", process.stdout, re.MULTILINE).group() + assert match=="E ModuleNotFoundError: No module named 'torch'" + result_file.unlink(missing_ok=True) From 3f6547d877f47af5c9ab1038ef8e4002bd8b6f83 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 15:29:45 -0700 Subject: [PATCH 5/8] compile regex pattern --- codeflash/code_utils/code_utils.py | 2 ++ codeflash/optimization/function_optimizer.py | 4 ++-- tests/test_test_runner.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 4c167fb69..8ec6ff344 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -15,6 +15,8 @@ from codeflash.cli_cmds.console import logger from codeflash.code_utils.config_parser import find_pyproject_toml +ImportErrorPattern = re.compile(r"^.*ModuleNotFoundError.*$", re.MULTILINE) + @contextmanager def custom_addopts() -> None: diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 0c9edfdee..12aeff3fa 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -3,7 +3,6 @@ import ast import concurrent.futures import os -import re import subprocess import time import uuid @@ -24,6 +23,7 @@ from codeflash.code_utils import env_utils from codeflash.code_utils.code_replacer import replace_function_definitions_in_module from codeflash.code_utils.code_utils import ( + ImportErrorPattern, cleanup_paths, file_name_from_test_module_name, get_run_tmp_file, @@ -1196,7 +1196,7 @@ def run_and_parse_tests( if "ModuleNotFoundError" in run_result.stdout: from rich.text import Text - match = re.search(r"^.*ModuleNotFoundError.*$", run_result.stdout, re.MULTILINE).group() + match = ImportErrorPattern.search(run_result.stdout).group() panel = Panel(Text.from_markup(f"⚠️ {match} ", style="bold red"), expand=False) console.print(panel) if testing_type in {TestingMode.BEHAVIOR, TestingMode.PERFORMANCE}: diff --git a/tests/test_test_runner.py b/tests/test_test_runner.py index ef7e69968..41b09ca35 100644 --- a/tests/test_test_runner.py +++ b/tests/test_test_runner.py @@ -4,6 +4,7 @@ import tempfile from pathlib import Path +from codeflash.code_utils.code_utils import ImportErrorPattern from codeflash.models.models import TestFile, TestFiles, TestType from codeflash.verification.parse_test_output import parse_test_xml from codeflash.verification.test_runner import run_behavioral_tests @@ -143,6 +144,6 @@ def test_sort(): results = parse_test_xml( test_xml_file_path=result_file, test_files=test_files, test_config=config, run_result=process ) - match = re.search(r"^.*ModuleNotFoundError.*$", process.stdout, re.MULTILINE).group() + match = ImportErrorPattern.search(process.stdout).group() assert match=="E ModuleNotFoundError: No module named 'torch'" result_file.unlink(missing_ok=True) From 045f91403814ae9d297323b73ef245ab4806fd04 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 15:54:38 -0700 Subject: [PATCH 6/8] display during test discovery too --- codeflash/discovery/discover_unit_tests.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/codeflash/discovery/discover_unit_tests.py b/codeflash/discovery/discover_unit_tests.py index b76e63a91..1fd86acce 100644 --- a/codeflash/discovery/discover_unit_tests.py +++ b/codeflash/discovery/discover_unit_tests.py @@ -14,9 +14,16 @@ import pytest from pydantic.dataclasses import dataclass +from rich.panel import Panel +from rich.text import Text from codeflash.cli_cmds.console import console, logger, test_files_progress_bar -from codeflash.code_utils.code_utils import custom_addopts, get_run_tmp_file, module_name_from_file_path +from codeflash.code_utils.code_utils import ( + ImportErrorPattern, + custom_addopts, + get_run_tmp_file, + module_name_from_file_path, +) from codeflash.code_utils.compat import SAFE_SYS_EXECUTABLE, codeflash_cache_db from codeflash.models.models import CodePosition, FunctionCalledInTest, TestsInFile, TestType @@ -180,6 +187,10 @@ def discover_tests_pytest( logger.warning( f"Failed to collect tests. Pytest Exit code: {exitcode}={pytest.ExitCode(exitcode).name}\n {error_section}" ) + if "ModuleNotFoundError" in result.stdout: + match = ImportErrorPattern.search(result.stdout).group() + panel = Panel(Text.from_markup(f"⚠️ {match} ", style="bold red"), expand=False) + console.print(panel) elif 0 <= exitcode <= 5: logger.warning(f"Failed to collect tests. Pytest Exit code: {exitcode}={pytest.ExitCode(exitcode).name}") From 4ce5f96f8096775cd97a5c5fedf8b5fb7c65b80d Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 16:19:08 -0700 Subject: [PATCH 7/8] only display the part from Modulenotfound onwards --- codeflash/code_utils/code_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 8ec6ff344..5fc9bd9e9 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -15,7 +15,7 @@ from codeflash.cli_cmds.console import logger from codeflash.code_utils.config_parser import find_pyproject_toml -ImportErrorPattern = re.compile(r"^.*ModuleNotFoundError.*$", re.MULTILINE) +ImportErrorPattern = re.compile(r"ModuleNotFoundError.*$", re.MULTILINE) @contextmanager From 9d8e0a24d2163bc82937957dd69a252b6f7caf53 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 2 Jun 2025 16:26:09 -0700 Subject: [PATCH 8/8] modify test --- tests/test_test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_test_runner.py b/tests/test_test_runner.py index 41b09ca35..0e80b76e0 100644 --- a/tests/test_test_runner.py +++ b/tests/test_test_runner.py @@ -145,5 +145,5 @@ def test_sort(): test_xml_file_path=result_file, test_files=test_files, test_config=config, run_result=process ) match = ImportErrorPattern.search(process.stdout).group() - assert match=="E ModuleNotFoundError: No module named 'torch'" + assert match=="ModuleNotFoundError: No module named 'torch'" result_file.unlink(missing_ok=True)