From 77d102fefdc86b11fea0a767d3130c6740b0d6df Mon Sep 17 00:00:00 2001 From: Codeflash Bot Date: Fri, 17 Oct 2025 18:30:56 +0300 Subject: [PATCH 1/4] log the test results errors to the lsp logs --- codeflash/code_utils/code_utils.py | 16 ++++++++++++++++ codeflash/optimization/function_optimizer.py | 14 ++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 1bc4d66bb..233ac9e47 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -353,3 +353,19 @@ def exit_with_message(message: str, *, error_on_exit: bool = False) -> None: paneled_text(message, panel_args={"style": "red"}) sys.exit(1 if error_on_exit else 0) + + +def extract_unique_errors(pytest_output: str) -> set[str]: + unique_errors = set() + + # Regex pattern to match error lines: + # - Start with 'E' followed by optional whitespace + # - Capture the actual error message + pattern = r"^E\s+(.*)$" + + for match in re.finditer(pattern, pytest_output, re.MULTILINE): + error_message = match.group(1).strip() + if error_message: + unique_errors.add(error_message) + + return unique_errors diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 8aaf11ec5..5e9037e34 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -34,6 +34,7 @@ cleanup_paths, create_rank_dictionary_compact, diff_length, + extract_unique_errors, file_name_from_test_module_name, get_run_tmp_file, module_name_from_file_path, @@ -1569,11 +1570,14 @@ def establish_original_code_baseline( ) if not behavioral_results: logger.warning( - f"force_lsp|Couldn't run any tests for original function {self.function_to_optimize.function_name}. SKIPPING OPTIMIZING THIS FUNCTION." + f"force_lsp|Couldn't run any tests for original function {self.function_to_optimize.function_name}. Skipping optimization." ) console.rule() return Failure("Failed to establish a baseline for the original code - bevhavioral tests failed.") if not coverage_critic(coverage_results, self.args.test_framework): + did_pass_all_tests = all(result.did_pass for result in behavioral_results) + if not did_pass_all_tests: + return Failure("Tests failed to pass for the original code.") return Failure( f"Test coverage is {coverage_results.coverage}%, which is below the required threshold of {COVERAGE_THRESHOLD}%." ) @@ -1601,7 +1605,7 @@ def establish_original_code_baseline( ) try: - benchmarking_results, _ = self.run_and_parse_tests( + benchmarking_results = self.run_and_parse_tests( testing_type=TestingMode.PERFORMANCE, test_env=test_env, test_files=self.test_files, @@ -1937,6 +1941,12 @@ def run_and_parse_tests( f"stdout: {run_result.stdout}\n" f"stderr: {run_result.stderr}\n" ) + + if is_LSP_enabled(): + unique_errors = extract_unique_errors(run_result.stdout) + if unique_errors: + lsp_log(LspCodeMessage(code="\n".join(unique_errors), file_name="errors")) + if "ModuleNotFoundError" in run_result.stdout: from rich.text import Text From ff924f449d72eb50c1abcde9eb7765c1d2a2d06a Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 21 Oct 2025 13:45:29 +0300 Subject: [PATCH 2/4] fix --- codeflash/optimization/function_optimizer.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 443eee28e..c44751fff 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -30,7 +30,6 @@ replace_function_definitions_in_module, ) from codeflash.code_utils.code_utils import ( - ImportErrorPattern, cleanup_paths, create_rank_dictionary_compact, diff_length, @@ -1612,7 +1611,7 @@ def establish_original_code_baseline( ) try: - benchmarking_results = self.run_and_parse_tests( + benchmarking_results, _ = self.run_and_parse_tests( testing_type=TestingMode.PERFORMANCE, test_env=test_env, test_files=self.test_files, @@ -1949,17 +1948,18 @@ def run_and_parse_tests( f"stderr: {run_result.stderr}\n" ) - if is_LSP_enabled(): - unique_errors = extract_unique_errors(run_result.stdout) - if unique_errors: - lsp_log(LspCodeMessage(code="\n".join(unique_errors), file_name="errors")) + unique_errors = extract_unique_errors(run_result.stdout) - if "ModuleNotFoundError" in run_result.stdout: + if unique_errors: from rich.text import Text - match = ImportErrorPattern.search(run_result.stdout).group() - panel = Panel(Text.from_markup(f"⚠️ {match} ", style="bold red"), expand=False) - console.print(panel) + for error in unique_errors: + if is_LSP_enabled(): + lsp_log(LspCodeMessage(code=error, file_name="errors")) + else: + panel = Panel(Text.from_markup(f"⚠️ {error} ", style="bold red"), expand=False) + console.print(panel) + if testing_type in {TestingMode.BEHAVIOR, TestingMode.PERFORMANCE}: results, coverage_results = parse_test_results( test_xml_path=result_file_path, From b8f30939bc9d1aea35bcaa02456a33387255afa6 Mon Sep 17 00:00:00 2001 From: mohammed ahmed <64513301+mohammedahmed18@users.noreply.github.com> Date: Tue, 21 Oct 2025 14:51:03 +0300 Subject: [PATCH 3/4] Apply suggestion from @codeflash-ai[bot] Co-authored-by: codeflash-ai[bot] <148906541+codeflash-ai[bot]@users.noreply.github.com> --- codeflash/code_utils/code_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 827a98787..85e543a79 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -365,8 +365,8 @@ def extract_unique_errors(pytest_output: str) -> set[str]: # - Capture the actual error message pattern = r"^E\s+(.*)$" - for match in re.finditer(pattern, pytest_output, re.MULTILINE): - error_message = match.group(1).strip() + for error_message in re.findall(pattern, pytest_output, re.MULTILINE): + error_message = error_message.strip() if error_message: unique_errors.add(error_message) From b2bb31ebfa378fd8eaa6040ea6ed610e3fd8cdc7 Mon Sep 17 00:00:00 2001 From: ali Date: Tue, 21 Oct 2025 14:53:37 +0300 Subject: [PATCH 4/4] ignore ruff lint warning --- codeflash/code_utils/code_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 85e543a79..507b5a169 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -366,7 +366,7 @@ def extract_unique_errors(pytest_output: str) -> set[str]: pattern = r"^E\s+(.*)$" for error_message in re.findall(pattern, pytest_output, re.MULTILINE): - error_message = error_message.strip() + error_message = error_message.strip() # noqa: PLW2901 if error_message: unique_errors.add(error_message)