Lint repo

.github/workflows/system.yml

@@ -39,7 +39,7 @@ jobs:
           uv run commit0 evaluate --reference  --rebuild
           uv run commit0 evaluate --reference
       - name: Lint
-        run: uv run commit0 lint commit0/harness/lint.py
+        run: uv run commit0 lint commit0/harness/
       - name: Save
         env:
           GITHUB_TOKEN: ${{ secrets.MY_GITHUB_TOKEN }}

agent/run_agent.py

@@ -115,14 +115,17 @@ def run_agent_for_repo(
             message = get_message(
                 agent_config, repo_path, test_dir=example["test"]["test_dir"]
             )
+            agent_config_log_file = os.path.abspath(
+                RUN_AIDER_LOG_DIR / "no_tests" / ".agent.yaml"
+            )
+            os.makedirs(os.path.dirname(agent_config_log_file), exist_ok=True)
+            # write agent_config to .agent.yaml
+            with open(agent_config_log_file, "w") as agent_config_file:
+                yaml.dump(agent_config, agent_config_file)
+
             for f in target_edit_files:
                 file_name = f.replace(".py", "").replace("/", "__")
                 log_dir = RUN_AIDER_LOG_DIR / "no_tests" / file_name
-                # write agent_config to .agent.yaml
-                with open(
-                    RUN_AIDER_LOG_DIR / "no_tests" / file_name / ".agent.yaml", "w"
-                ) as agent_config_file:
-                    yaml.dump(agent_config, agent_config_file)
                 lint_cmd = get_lint_cmd(local_repo, agent_config.use_lint_info)
                 agent.run(message, "", lint_cmd, [f], log_dir)
 

commit0/cli.py

@@ -1,6 +1,6 @@
 import typer
 from pathlib import Path
-from typing import List, Union
+from typing import Union
 from typing_extensions import Annotated
 import commit0.harness.run_pytest_ids
 import commit0.harness.get_pytest_ids
@@ -221,7 +221,10 @@ def test(
     timeout: int = typer.Option(1800, help="Timeout for tests in seconds"),
     num_cpus: int = typer.Option(1, help="Number of CPUs to use"),
     reference: Annotated[
-        bool, typer.Option("--reference", help="Test the reference commit.")
+        bool, typer.Option("--reference", help="Test the reference commit")
+    ] = False,
+    coverage: Annotated[
+        bool, typer.Option("--coverage", help="Whether to get coverage information")
     ] = False,
     rebuild: bool = typer.Option(
         False, "--rebuild", help="Whether to rebuild an image"
@@ -266,6 +269,7 @@ def test(
         repo_or_repo_path,
         branch,  # type: ignore
         test_ids,
+        coverage,
         backend,
         timeout,
         num_cpus,
@@ -286,6 +290,9 @@ def evaluate(
     reference: Annotated[
         bool, typer.Option("--reference", help="Evaluate the reference commit.")
     ] = False,
+    coverage: Annotated[
+        bool, typer.Option("--coverage", help="Whether to get coverage information")
+    ] = False,
     commit0_dot_file_path: str = typer.Option(
         ".commit0.yaml",
         help="Path to the commit0 dot file, where the setup config is stored",
@@ -309,6 +316,7 @@ def evaluate(
         commit0_config["repo_split"],
         commit0_config["base_dir"],
         branch,
+        coverage,
         backend,
         timeout,
         num_cpus,
@@ -319,17 +327,24 @@ def evaluate(
 
 @commit0_app.command()
 def lint(
-    files: List[Path] = typer.Argument(..., help="Files to lint."),
+    repo_or_repo_dir: str = typer.Argument(
+        ..., help="Directory of the repository to test"
+    ),
+    commit0_dot_file_path: str = typer.Option(
+        ".commit0.yaml",
+        help="Path to the commit0 dot file, where the setup config is stored",
+    ),
 ) -> None:
     """Lint given files if provided, otherwise lint all files in the base directory."""
     check_commit0_path()
-    for path in files:
-        if not path.is_file():
-            raise FileNotFoundError(f"File not found: {str(path)}")
-    typer.echo(
-        f"Linting specific files: {', '.join(highlight(str(file), Colors.ORANGE) for file in files)}"
+    commit0_config = read_commit0_dot_file(commit0_dot_file_path)
+    typer.echo(f"Linting repo: {highlight(str(repo_or_repo_dir), Colors.ORANGE)}")
+    commit0.harness.lint.main(
+        commit0_config["dataset_name"],
+        commit0_config["dataset_split"],
+        repo_or_repo_dir,
+        commit0_config["base_dir"],
     )
-    commit0.harness.lint.main(files)
 
 
 @commit0_app.command()

commit0/harness/constants.py

@@ -9,6 +9,7 @@ class RepoInstance(TypedDict):
     reference_commit: str
     setup: dict
     test: Dict[str, str]
+    src_dir: str
 
 
 class Files(TypedDict):

commit0/harness/evaluate.py

@@ -24,6 +24,7 @@ def main(
     repo_split: str,
     base_dir: str,
     branch: Union[str, None],
+    coverage: bool,
     backend: str,
     timeout: int,
     num_cpus: int,
@@ -58,6 +59,7 @@ def main(
                     repo,
                     branch,
                     test_dir,
+                    coverage,
                     backend,
                     timeout,
                     num_cpus,

commit0/harness/execution_context.py

@@ -163,7 +163,7 @@ def __init__(
 
         # the image must exist on dockerhub
         reponame = spec.repo.split("/")[-1]
-        image_name = f"wentingzhao/{reponame}:latest".lower()
+        image_name = f"wentingzhao/{reponame}:v0".lower()
         image = modal.Image.from_registry(image_name, force_build=rebuild_image)
         if files_to_copy:
             for _, f in files_to_copy.items():

commit0/harness/get_pytest_ids.py

@@ -1,20 +1,14 @@
-import tarfile
+import bz2
 from typing import List
 
 
 def main(repo: str, verbose: int) -> List[str]:
     repo = repo.lower()
     repo = repo.replace(".", "-")
-    out = ""
-    with tarfile.open(f"commit0/data/test_ids/{repo}.tar.bz2", "r:bz2") as tar:
-        for member in tar.getmembers():
-            if member.isfile():
-                file = tar.extractfile(member)
-                if file:
-                    content = file.read().decode("utf-8")
-                    out += content
-                    if verbose:
-                        print(content)
+    with bz2.open(f"commit0/data/test_ids/{repo}.bz2", "rt") as f:
+        out = f.read()
+    if verbose:
+        print(out)
     out = out.split("\n")
     return out
 

commit0/harness/lint.py

@@ -1,7 +1,13 @@
 import subprocess
 import sys
+import os
+from datasets import load_dataset
 from pathlib import Path
-from typing import List
+from typing import Iterator
+
+from commit0.harness.constants import (
+    RepoInstance,
+)
 
 
 config = """repos:
@@ -28,7 +34,38 @@
     - id: pyright"""
 
 
-def main(files: List[Path]) -> None:
+def main(
+    dataset_name: str, dataset_split: str, repo_or_repo_dir: str, base_dir: str
+) -> None:
+    dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
+    example = None
+    repo_name = None
+    for example in dataset:
+        repo_name = example["repo"].split("/")[-1]
+        if repo_or_repo_dir.endswith("/"):
+            repo_or_repo_dir = repo_or_repo_dir[:-1]
+        if repo_name in os.path.basename(repo_or_repo_dir):
+            break
+    assert example is not None, "No example available"
+    assert repo_name is not None, "No repo available"
+
+    repo_dir = os.path.join(base_dir, repo_name)
+    if os.path.isdir(repo_or_repo_dir):
+        repo = repo_or_repo_dir
+    elif os.path.isdir(repo_dir):
+        repo = repo_dir
+    else:
+        raise Exception(
+            f"Neither {repo_dir} nor {repo_or_repo_dir} is a valid path.\nUsage: commit0 lint {{repo_or_repo_dir}}"
+        )
+
+    files = []
+    repo = os.path.join(repo, example["src_dir"])
+    for root, dirs, fs in os.walk(repo):
+        for file in fs:
+            if file.endswith(".py"):
+                files.append(os.path.join(root, file))
+
     config_file = Path(".commit0.pre-commit-config.yaml")
     if not config_file.is_file():
         config_file.write_text(config)

commit0/harness/run_pytest_ids.py

@@ -34,6 +34,7 @@ def main(
     repo_or_repo_dir: str,
     branch: str,
     test_ids: str,
+    coverage: bool,
     backend: str,
     timeout: int,
     num_cpus: int,
@@ -113,7 +114,11 @@ def main(
     patch_file.write_text(patch)
 
     # make eval file
-    eval_script = spec.eval_script.format(test_ids=test_ids)
+    if coverage:
+        coverage_text = f" --cov={example['src_dir']} --cov-branch --cov-report json"
+    else:
+        coverage_text = ""
+    eval_script = spec.eval_script.format(test_ids=test_ids, coverage=coverage_text)
     eval_file = Path(log_dir / "eval.sh")
     eval_file.write_text(eval_script)
 
@@ -134,10 +139,11 @@ def main(
     )
     files_to_collect = [
         "report.json",
-        "coverage.json",
         "pytest_exit_code.txt",
         "test_output.txt",
     ]
+    if coverage:
+        files_to_collect.append("coverage.json")
 
     try:
         with execution_context(

commit0/harness/spec.py

@@ -54,13 +54,13 @@ def repo_image_key(self) -> str:
         repo = self.repo.split("/")[-1]
         # this is the image name created locally
         # once this image created, it will be tagged with repo_image_tag
-        return f"commit0.repo.{repo}.{val}:latest".lower()
+        return f"commit0.repo.{repo}.{val}:v0".lower()
 
     @property
     def repo_image_tag(self) -> str:
         """Repo image tag that will be used throughout."""
         repo = self.repo.split("/")[-1]
-        return f"wentingzhao/{repo}:latest".lower()
+        return f"wentingzhao/{repo}:v0".lower()
 
     def get_container_name(self, run_id: Optional[str] = None) -> str:
         repo = self.repo.split("/")[-1]
@@ -146,7 +146,7 @@ def make_repo_script_list(instance: RepoInstance, repo_directory: str) -> list[s
             )
         setup_commands.append(install)
     setup_commands.append(
-        "uv pip install pytest pytest-cov coverage pytest-json-report"
+        "uv pip install -U pytest pytest-cov coverage pytest-json-report"
     )
     setup_commands.append(f"git reset --hard {base_commit}")
     return setup_commands
@@ -160,10 +160,8 @@ def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[s
         f"git reset --hard {instance['base_commit']}",
         "git apply --allow-empty -v /patch.diff",
         "git status",
-        f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors --cov=. --cov-branch --cov-report json {{test_ids}} > test_output.txt 2>&1",
+        f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors{{coverage}} {{test_ids}} > test_output.txt 2>&1",
         "echo $? > pytest_exit_code.txt",
-        f"git reset --hard {instance['base_commit']}",
-        "git status",
     ]
     return eval_script_list