From 4e5b9334db83bae47b05d3a0295af0628d6c94a5 Mon Sep 17 00:00:00 2001 From: Aseem Saxena Date: Thu, 24 Apr 2025 17:26:50 -0700 Subject: [PATCH 1/3] fix high speedups for certain functions --- codeflash/models/models.py | 1 + codeflash/verification/equivalence.py | 1 + 2 files changed, 2 insertions(+) diff --git a/codeflash/models/models.py b/codeflash/models/models.py index ddaccd16e..1e27c8a96 100644 --- a/codeflash/models/models.py +++ b/codeflash/models/models.py @@ -561,6 +561,7 @@ def total_passed_runtime(self) -> int: :return: The runtime in nanoseconds. """ + #TODO this doesn't look at the interesection of tests of baseline and original return sum( [min(usable_runtime_data) for _, usable_runtime_data in self.usable_runtime_data_by_test_case().items()] ) diff --git a/codeflash/verification/equivalence.py b/codeflash/verification/equivalence.py index 67b9de439..af5ea3a70 100644 --- a/codeflash/verification/equivalence.py +++ b/codeflash/verification/equivalence.py @@ -70,6 +70,7 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR are_equal = False break +# TODO: add more tests, regression and replay if original_test_result.test_type in {TestType.EXISTING_UNIT_TEST, TestType.CONCOLIC_COVERAGE_TEST} and ( cdd_test_result.did_pass != original_test_result.did_pass ): From f39465acbd9b995ad77c2014646f57400c17c6de Mon Sep 17 00:00:00 2001 From: Aseem Saxena Date: Thu, 24 Apr 2025 19:14:15 -0700 Subject: [PATCH 2/3] quick fix --- codeflash/models/models.py | 2 +- codeflash/verification/equivalence.py | 2 +- tests/test_comparator.py | 88 +++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/codeflash/models/models.py b/codeflash/models/models.py index 1e27c8a96..aabf377df 100644 --- a/codeflash/models/models.py +++ b/codeflash/models/models.py @@ -561,7 +561,7 @@ def total_passed_runtime(self) -> int: :return: The runtime in nanoseconds. """ - #TODO this doesn't look at the interesection of tests of baseline and original + #TODO this doesn't look at the intersection of tests of baseline and original return sum( [min(usable_runtime_data) for _, usable_runtime_data in self.usable_runtime_data_by_test_case().items()] ) diff --git a/codeflash/verification/equivalence.py b/codeflash/verification/equivalence.py index af5ea3a70..e26e4ad81 100644 --- a/codeflash/verification/equivalence.py +++ b/codeflash/verification/equivalence.py @@ -71,7 +71,7 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR break # TODO: add more tests, regression and replay - if original_test_result.test_type in {TestType.EXISTING_UNIT_TEST, TestType.CONCOLIC_COVERAGE_TEST} and ( + if original_test_result.test_type in {TestType.EXISTING_UNIT_TEST, TestType.CONCOLIC_COVERAGE_TEST, TestType.GENERATED_REGRESSION, TestType.REPLAY_TEST} and ( cdd_test_result.did_pass != original_test_result.did_pass ): are_equal = False diff --git a/tests/test_comparator.py b/tests/test_comparator.py index 0f8ace054..4a4d9f2b1 100644 --- a/tests/test_comparator.py +++ b/tests/test_comparator.py @@ -1009,6 +1009,94 @@ def test_compare_results_fn(): assert not compare_test_results(original_results, new_results_4) + new_results_5_baseline = TestResults() + new_results_5_baseline.add( + FunctionTestInvocation( + id=InvocationId( + test_module_path="test_module_path", + test_class_name="test_class_name", + test_function_name="test_function_name", + function_getting_tested="function_getting_tested", + iteration_id="0", + ), + file_name=Path("file_name"), + did_pass=True, + runtime=5, + test_framework="unittest", + test_type=TestType.GENERATED_REGRESSION, + return_value=5, + timed_out=False, + loop_index=1, + ) + ) + + new_results_5_opt = TestResults() + new_results_5_opt.add( + FunctionTestInvocation( + id=InvocationId( + test_module_path="test_module_path", + test_class_name="test_class_name", + test_function_name="test_function_name", + function_getting_tested="function_getting_tested", + iteration_id="0", + ), + file_name=Path("file_name"), + did_pass=False, + runtime=5, + test_framework="unittest", + test_type=TestType.GENERATED_REGRESSION, + return_value=5, + timed_out=False, + loop_index=1, + ) + ) + + assert not compare_test_results(new_results_5_baseline, new_results_5_opt) + + new_results_6_baseline = TestResults() + new_results_6_baseline.add( + FunctionTestInvocation( + id=InvocationId( + test_module_path="test_module_path", + test_class_name="test_class_name", + test_function_name="test_function_name", + function_getting_tested="function_getting_tested", + iteration_id="0", + ), + file_name=Path("file_name"), + did_pass=True, + runtime=5, + test_framework="unittest", + test_type=TestType.REPLAY_TEST, + return_value=5, + timed_out=False, + loop_index=1, + ) + ) + + new_results_6_opt = TestResults() + new_results_6_opt.add( + FunctionTestInvocation( + id=InvocationId( + test_module_path="test_module_path", + test_class_name="test_class_name", + test_function_name="test_function_name", + function_getting_tested="function_getting_tested", + iteration_id="0", + ), + file_name=Path("file_name"), + did_pass=False, + runtime=5, + test_framework="unittest", + test_type=TestType.REPLAY_TEST, + return_value=5, + timed_out=False, + loop_index=1, + ) + ) + + assert not compare_test_results(new_results_6_baseline, new_results_6_opt) + assert not compare_test_results(TestResults(), TestResults()) From dc29c297e1180f272dab7b73db5b9545ca547228 Mon Sep 17 00:00:00 2001 From: Aseem Saxena Date: Thu, 24 Apr 2025 19:19:04 -0700 Subject: [PATCH 3/3] Update equivalence.py --- codeflash/verification/equivalence.py | 1 - 1 file changed, 1 deletion(-) diff --git a/codeflash/verification/equivalence.py b/codeflash/verification/equivalence.py index e26e4ad81..b7ce6978a 100644 --- a/codeflash/verification/equivalence.py +++ b/codeflash/verification/equivalence.py @@ -70,7 +70,6 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR are_equal = False break -# TODO: add more tests, regression and replay if original_test_result.test_type in {TestType.EXISTING_UNIT_TEST, TestType.CONCOLIC_COVERAGE_TEST, TestType.GENERATED_REGRESSION, TestType.REPLAY_TEST} and ( cdd_test_result.did_pass != original_test_result.did_pass ):