add

Author: wenting-zhao

commit0/cli.py

@@ -221,7 +221,10 @@ def test(
     timeout: int = typer.Option(1800, help="Timeout for tests in seconds"),
     num_cpus: int = typer.Option(1, help="Number of CPUs to use"),
     reference: Annotated[
-        bool, typer.Option("--reference", help="Test the reference commit.")
+        bool, typer.Option("--reference", help="Test the reference commit")
+    ] = False,
+    coverage: Annotated[
+        bool, typer.Option("--coverage", help="Whether to get coverage information")
     ] = False,
     rebuild: bool = typer.Option(
         False, "--rebuild", help="Whether to rebuild an image"
@@ -266,6 +269,7 @@ def test(
         repo_or_repo_path,
         branch,  # type: ignore
         test_ids,
+        coverage,
         backend,
         timeout,
         num_cpus,
@@ -286,6 +290,9 @@ def evaluate(
     reference: Annotated[
         bool, typer.Option("--reference", help="Evaluate the reference commit.")
     ] = False,
+    coverage: Annotated[
+        bool, typer.Option("--coverage", help="Whether to get coverage information")
+    ] = False,
     commit0_dot_file_path: str = typer.Option(
         ".commit0.yaml",
         help="Path to the commit0 dot file, where the setup config is stored",
@@ -309,6 +316,7 @@ def evaluate(
         commit0_config["repo_split"],
         commit0_config["base_dir"],
         branch,
+        coverage,
         backend,
         timeout,
         num_cpus,

commit0/harness/evaluate.py

@@ -24,6 +24,7 @@ def main(
     repo_split: str,
     base_dir: str,
     branch: Union[str, None],
+    coverage: bool,
     backend: str,
     timeout: int,
     num_cpus: int,
@@ -58,6 +59,7 @@ def main(
                     repo,
                     branch,
                     test_dir,
+                    coverage,
                     backend,
                     timeout,
                     num_cpus,

commit0/harness/get_pytest_ids.py

@@ -1,20 +1,12 @@
-import tarfile
+import bz2
 from typing import List
 
 
 def main(repo: str, verbose: int) -> List[str]:
     repo = repo.lower()
     repo = repo.replace(".", "-")
-    out = ""
-    with tarfile.open(f"commit0/data/test_ids/{repo}.tar.bz2", "r:bz2") as tar:
-        for member in tar.getmembers():
-            if member.isfile():
-                file = tar.extractfile(member)
-                if file:
-                    content = file.read().decode("utf-8")
-                    out += content
-                    if verbose:
-                        print(content)
+    with bz2.open(f"commit0/data/test_ids/{repo}.bz2", 'rt') as f:
+        out = f.read()
     out = out.split("\n")
     return out
 

commit0/harness/run_pytest_ids.py

@@ -34,6 +34,7 @@ def main(
     repo_or_repo_dir: str,
     branch: str,
     test_ids: str,
+    coverage: bool,
     backend: str,
     timeout: int,
     num_cpus: int,
@@ -113,7 +114,11 @@ def main(
     patch_file.write_text(patch)
 
     # make eval file
-    eval_script = spec.eval_script.format(test_ids=test_ids)
+    if coverage:
+        coverage_text = f" --cov={example['src_dir']} --cov-branch --cov-report json"
+    else:
+        coverage_text = ""
+    eval_script = spec.eval_script.format(test_ids=test_ids, coverage=coverage_text)
     eval_file = Path(log_dir / "eval.sh")
     eval_file.write_text(eval_script)
 
@@ -134,10 +139,11 @@ def main(
     )
     files_to_collect = [
         "report.json",
-        "coverage.json",
         "pytest_exit_code.txt",
         "test_output.txt",
     ]
+    if coverage:
+        files_to_collect.append("coverage.json")
 
     try:
         with execution_context(

commit0/harness/spec.py

@@ -146,7 +146,7 @@ def make_repo_script_list(instance: RepoInstance, repo_directory: str) -> list[s
             )
         setup_commands.append(install)
     setup_commands.append(
-        "uv pip install pytest pytest-cov coverage pytest-json-report"
+        "uv pip install -U pytest pytest-cov coverage pytest-json-report"
     )
     setup_commands.append(f"git reset --hard {base_commit}")
     return setup_commands
@@ -160,10 +160,8 @@ def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[s
         f"git reset --hard {instance['base_commit']}",
         "git apply --allow-empty -v /patch.diff",
         "git status",
-        f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors --cov=. --cov-branch --cov-report json {{test_ids}} > test_output.txt 2>&1",
+        f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors{{coverage}} {{test_ids}} > test_output.txt 2>&1",
         "echo $? > pytest_exit_code.txt",
-        f"git reset --hard {instance['base_commit']}",
-        "git status",
     ]
     return eval_script_list