tmp

Author: nanjiangwill

agent/agent_utils.py

@@ -6,6 +6,8 @@
 from pathlib import Path
 from typing import List
 import fitz
+from import_deps import ModuleSet
+from graphlib import TopologicalSorter, CycleError
 import yaml
 
 from agent.class_types import AgentConfig
@@ -190,8 +192,46 @@ def _find_files_to_edit(base_dir: str, src_dir: str, test_dir: str) -> list[str]
     return files
 
 
-def get_target_edit_files(target_dir: str, src_dir: str, test_dir: str) -> list[str]:
+def ignore_cycles(graph: dict):
+    ts = TopologicalSorter(graph)
+    try:
+        return list(set(ts.static_order()))
+    except CycleError as e:
+        # print(f"Cycle detected: {e.args[1]}")
+        # You can either break the cycle by modifying the graph or handle it as needed.
+        # For now, let's just remove the first node in the cycle and try again.
+        cycle_nodes = e.args[1]
+        node_to_remove = cycle_nodes[0]
+        # print(f"Removing node {node_to_remove} to resolve cycle.")
+        graph.pop(node_to_remove, None)
+        return ignore_cycles(graph)
+
+
+def topological_sort_based_on_dependencies(pkg_paths: list[str]) -> list[str]:
+    """Topological sort based on dependencies."""
+    module_set = ModuleSet([str(p) for p in pkg_paths])
+
+    import_dependencies = {}
+    for path in sorted(module_set.by_path.keys()):
+        module_name = ".".join(module_set.by_path[path].fqn)
+        mod = module_set.by_name[module_name]
+        imports = module_set.get_imports(mod)
+        import_dependencies[path] = set([str(x) for x in imports])
+
+    import_dependencies_files = ignore_cycles(import_dependencies)
+
+    return import_dependencies_files
+
+
+def get_target_edit_files(
+    local_repo: git.Repo,
+    src_dir: str,
+    test_dir: str,
+    latest_commit: str,
+    reference_commit: str,
+) -> list[str]:
     """Find the files with functions with the pass statement."""
+    target_dir = local_repo.working_dir
     files = _find_files_to_edit(target_dir, src_dir, test_dir)
     filtered_files = []
     for file_path in files:
@@ -202,13 +242,33 @@ def get_target_edit_files(target_dir: str, src_dir: str, test_dir: str) -> list[
             if "    pass" in content:
                 filtered_files.append(file_path)
 
+    # Change to reference commit to get the correct dependencies
+    local_repo.git.checkout(reference_commit)
+
+    topological_sort_files = topological_sort_based_on_dependencies(filtered_files)
+    if len(topological_sort_files) != len(filtered_files):
+        if len(topological_sort_files) < len(filtered_files):
+            # Find the missing elements
+            missing_files = set(filtered_files) - set(topological_sort_files)
+            # Add the missing files to the end of the list
+            topological_sort_files = topological_sort_files + list(missing_files)
+        else:
+            raise ValueError(
+                "topological_sort_files should not be longer than filtered_files"
+            )
+    assert len(topological_sort_files) == len(
+        filtered_files
+    ), "all files should be included"
+
+    # change to latest commit
+    local_repo.git.checkout(latest_commit)
+
     # Remove the base_dir prefix
-    filtered_files = [
-        file.replace(target_dir, "").lstrip("/") for file in filtered_files
+    topological_sort_files = [
+        file.replace(target_dir, "").lstrip("/") for file in topological_sort_files
     ]
-    # Only keep python files
 
-    return filtered_files
+    return topological_sort_files
 
 
 def get_message(

agent/display.py

@@ -17,6 +17,8 @@
 from rich.align import Align
 from collections import OrderedDict
 from types import TracebackType
+import json
+from datetime import datetime
 
 
 class RepoBox:
@@ -404,3 +406,29 @@ def __exit__(
             f"{'Total':<30} {self.total_time_spent:>13.2f}s {total_files:>18} {total_money:>13.2f}$"
         )
         print("-" * 80)
+
+        # Write summary to JSON file
+
+        summary_data = {
+            "timestamp": datetime.now().isoformat(),
+            "total_time_spent": self.total_time_spent,
+            "total_files_processed": total_files,
+            "total_money_spent": total_money,
+            "repositories": [
+                {
+                    "name": repo_name,
+                    "time_spent": self.end_time_per_repo[repo_name]
+                    - self.start_time_per_repo[repo_name],
+                    "files_processed": self.total_files_per_repo[repo_name],
+                    "money_spent": sum(
+                        self.repo_money_spent.get(repo_name, {}).values()
+                    ),
+                }
+                for repo_name in self.end_time_per_repo
+            ],
+        }
+
+        with open("processing_summary.json", "w") as json_file:
+            json.dump(summary_data, json_file, indent=4)
+
+        print("\nSummary has been written to processing_summary.json")

agent/run_agent.py

@@ -66,13 +66,6 @@ def run_agent_for_repo(
     repo_path = os.path.join(repo_base_dir, repo_name)
     repo_path = os.path.abspath(repo_path)
 
-    target_edit_files = get_target_edit_files(
-        repo_path, example["src_dir"], example["test"]["test_dir"]
-    )
-    # Call the commit0 get-tests command to retrieve test files
-    test_files_str = get_tests(repo_name, verbose=0)
-    test_files = sorted(list(set([i.split(":")[0] for i in test_files_str])))
-
     try:
         local_repo = Repo(repo_path)
     except Exception:
@@ -90,7 +83,6 @@ def run_agent_for_repo(
     # # if branch_name is not provided, create a new branch name based on agent_config
     # if branch is None:
     #     branch = args2string(agent_config)
-
     create_branch(local_repo, branch, example["base_commit"])
 
     # in cases where the latest commit of branch is not commit 0
@@ -99,6 +91,17 @@ def run_agent_for_repo(
     if latest_commit.hexsha != example["base_commit"] and override_previous_changes:
         local_repo.git.reset("--hard", example["base_commit"])
 
+    target_edit_files = get_target_edit_files(
+        local_repo,
+        example["src_dir"],
+        example["test"]["test_dir"],
+        latest_commit,
+        example["reference_commit"],
+    )
+    # Call the commit0 get-tests command to retrieve test files
+    test_files_str = get_tests(repo_name, verbose=0)
+    test_files = sorted(list(set([i.split(":")[0] for i in test_files_str])))
+
     # prepare the log dir
     experiment_log_dir = (
         Path(log_dir)