From 9068bfff4d77f349aa58e582b9ebe37190682481 Mon Sep 17 00:00:00 2001 From: Aseem Saxena Date: Tue, 8 Apr 2025 19:17:54 -0700 Subject: [PATCH 1/5] works for ab testing --- codeflash/optimization/function_optimizer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 93def83c0..341030f8d 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -235,8 +235,8 @@ def optimize_function(self) -> Result[BestOptimization, str]: # request for new optimizations but don't block execution, check for completion later # adding to control and experiment set but with same traceid best_optimization = None - - for _u, candidates in enumerate([optimizations_set.control, optimizations_set.experiment]): + run_experiment = self.experiment_id is not None + for _u, (candidates,exp_type) in enumerate(zip([optimizations_set.control, optimizations_set.experiment],["EXP0","EXP1"])): if candidates is None: continue @@ -246,6 +246,8 @@ def optimize_function(self) -> Result[BestOptimization, str]: original_code_baseline=original_code_baseline, original_helper_code=original_helper_code, file_path_to_helper_classes=file_path_to_helper_classes, + exp_type=exp_type, + run_experiment=run_experiment, ) ph("cli-optimize-function-finished", {"function_trace_id": self.function_trace_id}) @@ -346,6 +348,8 @@ def determine_best_candidate( original_code_baseline: OriginalCodeBaseline, original_helper_code: dict[Path, str], file_path_to_helper_classes: dict[Path, set[str]], + exp_type: str, + run_experiment: bool ) -> BestOptimization | None: best_optimization: BestOptimization | None = None best_runtime_until_now = original_code_baseline.runtime @@ -367,7 +371,7 @@ def determine_best_candidate( self.aiservice_client.optimize_python_code_line_profiler, source_code=code_context.read_writable_code, dependency_code=code_context.read_only_context_code, - trace_id=self.function_trace_id, + trace_id=self.function_trace_id[:-4] + exp_type if run_experiment else self.function_trace_id, line_profiler_results=original_code_baseline.line_profile_results["str_out"], num_candidates=10, experiment_metadata=None, @@ -377,7 +381,6 @@ def determine_best_candidate( done = False original_len = len(candidates) while candidates: - # for candidate_index, candidate in enumerate(candidates, start=1): done = True if future_line_profile_results is None else future_line_profile_results.done() if done and (future_line_profile_results is not None): line_profile_results = future_line_profile_results.result() From 545b8ed265451651499d08c87dc284b8ac5472db Mon Sep 17 00:00:00 2001 From: Saurabh Misra Date: Tue, 8 Apr 2025 19:53:36 -0700 Subject: [PATCH 2/5] line profiler is now experiment aware --- codeflash/optimization/function_optimizer.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 341030f8d..6d5c0eeee 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -235,8 +235,7 @@ def optimize_function(self) -> Result[BestOptimization, str]: # request for new optimizations but don't block execution, check for completion later # adding to control and experiment set but with same traceid best_optimization = None - run_experiment = self.experiment_id is not None - for _u, (candidates,exp_type) in enumerate(zip([optimizations_set.control, optimizations_set.experiment],["EXP0","EXP1"])): + for _u, (candidates, exp_type) in enumerate(zip([optimizations_set.control, optimizations_set.experiment],["EXP0","EXP1"])): if candidates is None: continue @@ -247,7 +246,6 @@ def optimize_function(self) -> Result[BestOptimization, str]: original_helper_code=original_helper_code, file_path_to_helper_classes=file_path_to_helper_classes, exp_type=exp_type, - run_experiment=run_experiment, ) ph("cli-optimize-function-finished", {"function_trace_id": self.function_trace_id}) @@ -349,7 +347,6 @@ def determine_best_candidate( original_helper_code: dict[Path, str], file_path_to_helper_classes: dict[Path, set[str]], exp_type: str, - run_experiment: bool ) -> BestOptimization | None: best_optimization: BestOptimization | None = None best_runtime_until_now = original_code_baseline.runtime @@ -367,18 +364,18 @@ def determine_best_candidate( # Start a new thread for AI service request, start loop in main thread # check if aiservice request is complete, when it is complete, append result to the candidates list with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + ai_service_client = self.local_aiservice_client if self.experiment_id else self.aiservice_client future_line_profile_results = executor.submit( - self.aiservice_client.optimize_python_code_line_profiler, + ai_service_client.optimize_python_code_line_profiler, source_code=code_context.read_writable_code, dependency_code=code_context.read_only_context_code, - trace_id=self.function_trace_id[:-4] + exp_type if run_experiment else self.function_trace_id, + trace_id=self.function_trace_id[:-4] + exp_type if self.experiment_id else self.function_trace_id, line_profiler_results=original_code_baseline.line_profile_results["str_out"], num_candidates=10, - experiment_metadata=None, + experiment_metadata=ExperimentMetadata(id=self.experiment_id, group= "control" if exp_type == "EXP0" else "experiment") if self.experiment_id else None, ) try: candidate_index = 0 - done = False original_len = len(candidates) while candidates: done = True if future_line_profile_results is None else future_line_profile_results.done() From 9857c339a4490a15742a9b8846fc90337017d4fd Mon Sep 17 00:00:00 2001 From: Aseem Saxena Date: Wed, 9 Apr 2025 17:40:10 -0700 Subject: [PATCH 3/5] setting ai service correctly --- codeflash/optimization/function_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 6d5c0eeee..62b8a81c7 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -364,7 +364,7 @@ def determine_best_candidate( # Start a new thread for AI service request, start loop in main thread # check if aiservice request is complete, when it is complete, append result to the candidates list with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: - ai_service_client = self.local_aiservice_client if self.experiment_id else self.aiservice_client + ai_service_client = self.aiservice_client if exp_type=="EXP0" else self.local_aiservice_client future_line_profile_results = executor.submit( ai_service_client.optimize_python_code_line_profiler, source_code=code_context.read_writable_code, From 5fa386a7b55115b58c08a91d44ca0dbcaa8976e7 Mon Sep 17 00:00:00 2001 From: Aseem Saxena Date: Thu, 10 Apr 2025 15:50:29 -0700 Subject: [PATCH 4/5] sending the right trace id to avoid overwriting in the db --- codeflash/optimization/function_optimizer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index 62b8a81c7..817b35e00 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -247,7 +247,7 @@ def optimize_function(self) -> Result[BestOptimization, str]: file_path_to_helper_classes=file_path_to_helper_classes, exp_type=exp_type, ) - ph("cli-optimize-function-finished", {"function_trace_id": self.function_trace_id}) + ph("cli-optimize-function-finished", {"function_trace_id": self.function_trace_id[:-4] + exp_type if self.experiment_id else self.function_trace_id}) generated_tests = remove_functions_from_generated_tests( generated_tests=generated_tests, test_functions_to_remove=test_functions_to_remove @@ -271,7 +271,7 @@ def optimize_function(self) -> Result[BestOptimization, str]: file_path=self.function_to_optimize.file_path, ) - self.log_successful_optimization(explanation, generated_tests) + self.log_successful_optimization(explanation, generated_tests, exp_type) self.replace_function_and_helpers_with_optimized_code( code_context=code_context, optimized_code=best_optimization.candidate.source_code @@ -309,7 +309,7 @@ def optimize_function(self) -> Result[BestOptimization, str]: explanation=explanation, existing_tests_source=existing_tests, generated_original_test_source=generated_tests_str, - function_trace_id=self.function_trace_id, + function_trace_id=self.function_trace_id[:-4] + exp_type if self.experiment_id else self.function_trace_id, coverage_message=coverage_message, git_remote=self.args.git_remote, ) @@ -477,8 +477,8 @@ def determine_best_candidate( logger.exception(f"Optimization interrupted: {e}") raise - self.aiservice_client.log_results( - function_trace_id=self.function_trace_id, + ai_service_client.log_results( + function_trace_id=self.function_trace_id[:-4] + exp_type if self.experiment_id else self.function_trace_id, speedup_ratio=speedup_ratios, original_runtime=original_code_baseline.runtime, optimized_runtime=optimized_runtimes, @@ -486,7 +486,7 @@ def determine_best_candidate( ) return best_optimization - def log_successful_optimization(self, explanation: Explanation, generated_tests: GeneratedTestsList) -> None: + def log_successful_optimization(self, explanation: Explanation, generated_tests: GeneratedTestsList, exp_type: str) -> None: explanation_panel = Panel( f"⚡️ Optimization successful! 📄 {self.function_to_optimize.qualified_name} in {explanation.file_path}\n" f"📈 {explanation.perf_improvement_line}\n" @@ -512,7 +512,7 @@ def log_successful_optimization(self, explanation: Explanation, generated_tests: ph( "cli-optimize-success", { - "function_trace_id": self.function_trace_id, + "function_trace_id": self.function_trace_id[:-4] + exp_type if self.experiment_id else self.function_trace_id, "speedup_x": explanation.speedup_x, "speedup_pct": explanation.speedup_pct, "best_runtime": explanation.best_runtime_ns, From 959b8af2e5e49d2406bc66dd49b249619fdc952c Mon Sep 17 00:00:00 2001 From: Aseem Saxena Date: Mon, 21 Apr 2025 13:55:13 -0700 Subject: [PATCH 5/5] correct length calculation --- codeflash/optimization/function_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeflash/optimization/function_optimizer.py b/codeflash/optimization/function_optimizer.py index ffc721dbe..6ad25bc0e 100644 --- a/codeflash/optimization/function_optimizer.py +++ b/codeflash/optimization/function_optimizer.py @@ -397,7 +397,7 @@ def determine_best_candidate( if done and (future_line_profile_results is not None): line_profile_results = future_line_profile_results.result() candidates.extend(line_profile_results) - original_len += len(candidates) + original_len += len(line_profile_results) logger.info( f"Added results from line profiler to candidates, total candidates now: {original_len}" )