fix small display error and update get file method

Author: nanjiangwill

agent/README.md

@@ -37,3 +37,24 @@ python baselines/run_aider.py
 - `max_unit_tests_info_length`: Max length of unit tests info. Default: `10000`.
 - `max_reference_info_length`: Max length of reference info. Default: `10000`.
 - `max_lint_info_length`: Max length of lint info. Default: `10000`.
+
+
+
+
+
+
+
+
+
+
+Error Section
+
+
+
+Running the agent
+
+Run with Tmux!
+
+process_max_worker set to 3....
+
+currently not handling file with more than 1500 lines...

agent/agent_utils.py

@@ -119,24 +119,93 @@ def get_file_info(file_path: Path, prefix: str = "") -> str:
     return "\n".join(filter(None, tree_string))
 
 
-def get_target_edit_files(target_dir: str, src_prefix: str) -> list[str]:
+def collect_test_files(directory: str) -> list[str]:
+    """Collect all the test files in the directory."""
+    test_files = []
+    subdirs = []
+
+    # Walk through the directory
+    for root, dirs, files in os.walk(directory):
+        if root.endswith("/"):
+            root = root[:-1]
+        # Check if 'test' is part of the folder name
+        if (
+            "test" in os.path.basename(root).lower()
+            or os.path.basename(root) in subdirs
+        ):
+            for file in files:
+                # Process only Python files
+                if file.endswith(".py"):
+                    file_path = os.path.join(root, file)
+                    test_files.append(file_path)
+            for d in dirs:
+                subdirs.append(d)
+
+    return test_files
+
+
+def collect_python_files(directory: str) -> list[str]:
+    """List to store all the .py filenames"""
+    python_files = []
+
+    # Walk through the directory recursively
+    for root, _, files in os.walk(directory):
+        for file in files:
+            # Check if the file ends with '.py'
+            if file.endswith(".py"):
+                file_path = os.path.join(root, file)
+                python_files.append(file_path)
+
+    return python_files
+
+
+def _find_files_to_edit(base_dir: str, src_dir: str, test_dir: str) -> list[str]:
+    """Identify files to remove content by heuristics.
+    We assume source code is under [lib]/[lib] or [lib]/src.
+    We exclude test code. This function would not work
+    if test code doesn't have its own directory.
+
+    Args:
+    ----
+        base_dir (str): The path to local library.
+        src_dir (str): The directory containing source code.
+        test_dir (str): The directory containing test code.
+
+    Returns:
+    -------
+        list[str]: A list of files to be edited.
+
+    """
+    files = collect_python_files(os.path.join(base_dir, src_dir))
+    test_files = collect_test_files(os.path.join(base_dir, test_dir))
+    files = list(set(files) - set(test_files))
+
+    # don't edit __init__ files
+    files = [f for f in files if "__init__" not in f]
+    # don't edit __main__ files
+    files = [f for f in files if "__main__" not in f]
+    # don't edit confest.py files
+    files = [f for f in files if "conftest.py" not in f]
+    return files
+
+
+def get_target_edit_files(target_dir: str, src_dir: str, test_dir: str) -> list[str]:
     """Find the files with functions with the pass statement."""
-    files = []
-    for root, _, filenames in os.walk(target_dir):
-        for filename in filenames:
-            if filename.endswith(".py"):
-                file_path = os.path.join(root, filename)
-                with open(file_path, "r", encoding="utf-8", errors="ignore") as file:
-                    if "    pass" in file.read():
-                        files.append(file_path)
+    files = _find_files_to_edit(target_dir, src_dir, test_dir)
+    filtered_files = []
+    for file_path in files:
+        with open(file_path, "r", encoding="utf-8", errors="ignore") as file:
+            content = file.read()
+            if len(content.splitlines()) < 1500:
+                filtered_files.append(file_path)
 
     # Remove the base_dir prefix
-    files = [file.replace(target_dir, "").lstrip("/") for file in files]
-    files = [src_prefix + file for file in files]
+    filtered_files = [
+        file.replace(target_dir, "").lstrip("/") for file in filtered_files
+    ]
     # Only keep python files
-    files = [file for file in files if file.endswith(".py")]
 
-    return files
+    return filtered_files
 
 
 def get_message(

agent/run_agent.py

@@ -61,17 +61,15 @@ def run_agent_for_repo(
     original_repo_name = repo_name
     update_queue.put(("start_repo", (original_repo_name, 0)))
 
-    repo_name = repo_name.lower()
-    repo_name = repo_name.replace(".", "-")
+    # repo_name = repo_name.lower()
+    # repo_name = repo_name.replace(".", "-")
 
     repo_path = os.path.join(repo_base_dir, repo_name)
     repo_path = os.path.abspath(repo_path)
 
-    # TODO: remove this to make web3.py work
-    if repo_name == "web3-py":
-        repo_path = repo_path.replace("web3-py", "web3.py")
-
-    src_dir = os.path.join(repo_path, example["src_dir"])
+    # # TODO: remove this to make web3.py work
+    # if repo_name == "web3-py":
+    #     repo_path = repo_path.replace("web3-py", "web3.py")
 
     try:
         local_repo = Repo(repo_path)
@@ -121,7 +119,7 @@ def run_agent_for_repo(
             raise ValueError("Invalid input")
 
         target_edit_files = get_target_edit_files(
-            src_dir, src_prefix=example["src_dir"]
+            repo_path, example["src_dir"], example["test"]["test_dir"]
         )
 
         if agent_config.run_tests:
@@ -133,7 +131,7 @@ def run_agent_for_repo(
             # when unit test feedback is available, iterate over test files
             for test_file in test_files:
                 update_queue.put(("set_current_file", (repo_name, test_file)))
-                test_cmd = f"python -m commit0 test {repo_path} {test_file} --branch {branch} --backend {backend} --commit0_dot_file_path {commit0_dot_file_path}"
+                test_cmd = f"python -m commit0 test {repo_path} {test_file} --branch {branch} --backend {backend} --commit0-dot-file-path {commit0_dot_file_path}"
                 test_file_name = test_file.replace(".py", "").replace("/", "__")
                 test_log_dir = experiment_log_dir / test_file_name
                 lint_cmd = get_lint_cmd(repo_name, agent_config.use_lint_info)

agent/run_agent_no_rich.py

@@ -60,8 +60,6 @@ def run_agent_for_repo(
     repo_path = os.path.join(repo_base_dir, repo_name)
     repo_path = os.path.abspath(repo_path)
 
-    src_dir = os.path.join(repo_path, example["src_dir"])
-
     try:
         local_repo = Repo(repo_path)
     except Exception:
@@ -109,7 +107,7 @@ def run_agent_for_repo(
             raise ValueError("Invalid input")
 
         target_edit_files = get_target_edit_files(
-            src_dir, src_prefix=example["src_dir"]
+            repo_path, example["src_dir"], example["test"]["test_dir"]
         )
 
         if agent_config.run_tests: