Fix test

agent/run_agent.py

@@ -115,14 +115,17 @@ def run_agent_for_repo(
             message = get_message(
                 agent_config, repo_path, test_dir=example["test"]["test_dir"]
             )
+            agent_config_log_file = os.path.abspath(
+                RUN_AIDER_LOG_DIR / "no_tests" / ".agent.yaml"
+            )
+            os.makedirs(os.path.dirname(agent_config_log_file), exist_ok=True)
+            # write agent_config to .agent.yaml
+            with open(agent_config_log_file, "w") as agent_config_file:
+                yaml.dump(agent_config, agent_config_file)
+
             for f in target_edit_files:
                 file_name = f.replace(".py", "").replace("/", "__")
                 log_dir = RUN_AIDER_LOG_DIR / "no_tests" / file_name
-                # write agent_config to .agent.yaml
-                with open(
-                    RUN_AIDER_LOG_DIR / "no_tests" / file_name / ".agent.yaml", "w"
-                ) as agent_config_file:
-                    yaml.dump(agent_config, agent_config_file)
                 lint_cmd = get_lint_cmd(local_repo, agent_config.use_lint_info)
                 agent.run(message, "", lint_cmd, [f], log_dir)
 

commit0/cli.py

@@ -221,7 +221,10 @@ def test(
     timeout: int = typer.Option(1800, help="Timeout for tests in seconds"),
     num_cpus: int = typer.Option(1, help="Number of CPUs to use"),
     reference: Annotated[
-        bool, typer.Option("--reference", help="Test the reference commit.")
+        bool, typer.Option("--reference", help="Test the reference commit")
+    ] = False,
+    coverage: Annotated[
+        bool, typer.Option("--coverage", help="Whether to get coverage information")
     ] = False,
     rebuild: bool = typer.Option(
         False, "--rebuild", help="Whether to rebuild an image"
@@ -266,6 +269,7 @@ def test(
         repo_or_repo_path,
         branch,  # type: ignore
         test_ids,
+        coverage,
         backend,
         timeout,
         num_cpus,
@@ -286,6 +290,9 @@ def evaluate(
     reference: Annotated[
         bool, typer.Option("--reference", help="Evaluate the reference commit.")
     ] = False,
+    coverage: Annotated[
+        bool, typer.Option("--coverage", help="Whether to get coverage information")
+    ] = False,
     commit0_dot_file_path: str = typer.Option(
         ".commit0.yaml",
         help="Path to the commit0 dot file, where the setup config is stored",
@@ -309,6 +316,7 @@ def evaluate(
         commit0_config["repo_split"],
         commit0_config["base_dir"],
         branch,
+        coverage,
         backend,
         timeout,
         num_cpus,

commit0/harness/constants.py

@@ -9,6 +9,7 @@ class RepoInstance(TypedDict):
     reference_commit: str
     setup: dict
     test: Dict[str, str]
+    src_dir: str
 
 
 class Files(TypedDict):

commit0/harness/evaluate.py

@@ -24,6 +24,7 @@ def main(
     repo_split: str,
     base_dir: str,
     branch: Union[str, None],
+    coverage: bool,
     backend: str,
     timeout: int,
     num_cpus: int,
@@ -58,6 +59,7 @@ def main(
                     repo,
                     branch,
                     test_dir,
+                    coverage,
                     backend,
                     timeout,
                     num_cpus,

commit0/harness/execution_context.py

@@ -163,7 +163,7 @@ def __init__(
 
         # the image must exist on dockerhub
         reponame = spec.repo.split("/")[-1]
-        image_name = f"wentingzhao/{reponame}:latest".lower()
+        image_name = f"wentingzhao/{reponame}:v0".lower()
         image = modal.Image.from_registry(image_name, force_build=rebuild_image)
         if files_to_copy:
             for _, f in files_to_copy.items():

commit0/harness/get_pytest_ids.py

@@ -1,20 +1,14 @@
-import tarfile
+import bz2
 from typing import List
 
 
 def main(repo: str, verbose: int) -> List[str]:
     repo = repo.lower()
     repo = repo.replace(".", "-")
-    out = ""
-    with tarfile.open(f"commit0/data/test_ids/{repo}.tar.bz2", "r:bz2") as tar:
-        for member in tar.getmembers():
-            if member.isfile():
-                file = tar.extractfile(member)
-                if file:
-                    content = file.read().decode("utf-8")
-                    out += content
-                    if verbose:
-                        print(content)
+    with bz2.open(f"commit0/data/test_ids/{repo}.bz2", "rt") as f:
+        out = f.read()
+    if verbose:
+        print(out)
     out = out.split("\n")
     return out
 

commit0/harness/run_pytest_ids.py

@@ -34,6 +34,7 @@ def main(
     repo_or_repo_dir: str,
     branch: str,
     test_ids: str,
+    coverage: bool,
     backend: str,
     timeout: int,
     num_cpus: int,
@@ -113,7 +114,11 @@ def main(
     patch_file.write_text(patch)
 
     # make eval file
-    eval_script = spec.eval_script.format(test_ids=test_ids)
+    if coverage:
+        coverage_text = f" --cov={example['src_dir']} --cov-branch --cov-report json"
+    else:
+        coverage_text = ""
+    eval_script = spec.eval_script.format(test_ids=test_ids, coverage=coverage_text)
     eval_file = Path(log_dir / "eval.sh")
     eval_file.write_text(eval_script)
 
@@ -134,10 +139,11 @@ def main(
     )
     files_to_collect = [
         "report.json",
-        "coverage.json",
         "pytest_exit_code.txt",
         "test_output.txt",
     ]
+    if coverage:
+        files_to_collect.append("coverage.json")
 
     try:
         with execution_context(

commit0/harness/spec.py

@@ -54,13 +54,13 @@ def repo_image_key(self) -> str:
         repo = self.repo.split("/")[-1]
         # this is the image name created locally
         # once this image created, it will be tagged with repo_image_tag
-        return f"commit0.repo.{repo}.{val}:latest".lower()
+        return f"commit0.repo.{repo}.{val}:v0".lower()
 
     @property
     def repo_image_tag(self) -> str:
         """Repo image tag that will be used throughout."""
         repo = self.repo.split("/")[-1]
-        return f"wentingzhao/{repo}:latest".lower()
+        return f"wentingzhao/{repo}:v0".lower()
 
     def get_container_name(self, run_id: Optional[str] = None) -> str:
         repo = self.repo.split("/")[-1]
@@ -146,7 +146,7 @@ def make_repo_script_list(instance: RepoInstance, repo_directory: str) -> list[s
             )
         setup_commands.append(install)
     setup_commands.append(
-        "uv pip install pytest pytest-cov coverage pytest-json-report"
+        "uv pip install -U pytest pytest-cov coverage pytest-json-report"
     )
     setup_commands.append(f"git reset --hard {base_commit}")
     return setup_commands
@@ -160,10 +160,8 @@ def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[s
         f"git reset --hard {instance['base_commit']}",
         "git apply --allow-empty -v /patch.diff",
         "git status",
-        f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors --cov=. --cov-branch --cov-report json {{test_ids}} > test_output.txt 2>&1",
+        f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors{{coverage}} {{test_ids}} > test_output.txt 2>&1",
         "echo $? > pytest_exit_code.txt",
-        f"git reset --hard {instance['base_commit']}",
-        "git status",
     ]
     return eval_script_list