diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index b1fb898..6cf32af 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -39,7 +39,7 @@ jobs: uv run commit0 evaluate --reference --rebuild uv run commit0 evaluate --reference - name: Lint - run: uv run commit0 lint commit0/harness/lint.py + run: uv run commit0 lint commit0/harness/ - name: Save env: GITHUB_TOKEN: ${{ secrets.MY_GITHUB_TOKEN }} diff --git a/agent/run_agent.py b/agent/run_agent.py index f03cc20..3d68eff 100644 --- a/agent/run_agent.py +++ b/agent/run_agent.py @@ -115,14 +115,17 @@ def run_agent_for_repo( message = get_message( agent_config, repo_path, test_dir=example["test"]["test_dir"] ) + agent_config_log_file = os.path.abspath( + RUN_AIDER_LOG_DIR / "no_tests" / ".agent.yaml" + ) + os.makedirs(os.path.dirname(agent_config_log_file), exist_ok=True) + # write agent_config to .agent.yaml + with open(agent_config_log_file, "w") as agent_config_file: + yaml.dump(agent_config, agent_config_file) + for f in target_edit_files: file_name = f.replace(".py", "").replace("/", "__") log_dir = RUN_AIDER_LOG_DIR / "no_tests" / file_name - # write agent_config to .agent.yaml - with open( - RUN_AIDER_LOG_DIR / "no_tests" / file_name / ".agent.yaml", "w" - ) as agent_config_file: - yaml.dump(agent_config, agent_config_file) lint_cmd = get_lint_cmd(local_repo, agent_config.use_lint_info) agent.run(message, "", lint_cmd, [f], log_dir) diff --git a/commit0/cli.py b/commit0/cli.py index c0d8596..9d49fcd 100644 --- a/commit0/cli.py +++ b/commit0/cli.py @@ -1,6 +1,6 @@ import typer from pathlib import Path -from typing import List, Union +from typing import Union from typing_extensions import Annotated import commit0.harness.run_pytest_ids import commit0.harness.get_pytest_ids @@ -221,7 +221,10 @@ def test( timeout: int = typer.Option(1800, help="Timeout for tests in seconds"), num_cpus: int = typer.Option(1, help="Number of CPUs to use"), reference: Annotated[ - bool, typer.Option("--reference", help="Test the reference commit.") + bool, typer.Option("--reference", help="Test the reference commit") + ] = False, + coverage: Annotated[ + bool, typer.Option("--coverage", help="Whether to get coverage information") ] = False, rebuild: bool = typer.Option( False, "--rebuild", help="Whether to rebuild an image" @@ -266,6 +269,7 @@ def test( repo_or_repo_path, branch, # type: ignore test_ids, + coverage, backend, timeout, num_cpus, @@ -286,6 +290,9 @@ def evaluate( reference: Annotated[ bool, typer.Option("--reference", help="Evaluate the reference commit.") ] = False, + coverage: Annotated[ + bool, typer.Option("--coverage", help="Whether to get coverage information") + ] = False, commit0_dot_file_path: str = typer.Option( ".commit0.yaml", help="Path to the commit0 dot file, where the setup config is stored", @@ -309,6 +316,7 @@ def evaluate( commit0_config["repo_split"], commit0_config["base_dir"], branch, + coverage, backend, timeout, num_cpus, @@ -319,17 +327,24 @@ def evaluate( @commit0_app.command() def lint( - files: List[Path] = typer.Argument(..., help="Files to lint."), + repo_or_repo_dir: str = typer.Argument( + ..., help="Directory of the repository to test" + ), + commit0_dot_file_path: str = typer.Option( + ".commit0.yaml", + help="Path to the commit0 dot file, where the setup config is stored", + ), ) -> None: """Lint given files if provided, otherwise lint all files in the base directory.""" check_commit0_path() - for path in files: - if not path.is_file(): - raise FileNotFoundError(f"File not found: {str(path)}") - typer.echo( - f"Linting specific files: {', '.join(highlight(str(file), Colors.ORANGE) for file in files)}" + commit0_config = read_commit0_dot_file(commit0_dot_file_path) + typer.echo(f"Linting repo: {highlight(str(repo_or_repo_dir), Colors.ORANGE)}") + commit0.harness.lint.main( + commit0_config["dataset_name"], + commit0_config["dataset_split"], + repo_or_repo_dir, + commit0_config["base_dir"], ) - commit0.harness.lint.main(files) @commit0_app.command() diff --git a/commit0/data/test_ids/attrs.bz2 b/commit0/data/test_ids/attrs.bz2 new file mode 100644 index 0000000..4336438 Binary files /dev/null and b/commit0/data/test_ids/attrs.bz2 differ diff --git a/commit0/data/test_ids/attrs.tar.bz2 b/commit0/data/test_ids/attrs.tar.bz2 deleted file mode 100644 index c451a90..0000000 Binary files a/commit0/data/test_ids/attrs.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/babel.bz2 b/commit0/data/test_ids/babel.bz2 new file mode 100644 index 0000000..37bc731 Binary files /dev/null and b/commit0/data/test_ids/babel.bz2 differ diff --git a/commit0/data/test_ids/babel.tar.bz2 b/commit0/data/test_ids/babel.tar.bz2 deleted file mode 100644 index 15552e9..0000000 Binary files a/commit0/data/test_ids/babel.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/bitstring.bz2 b/commit0/data/test_ids/bitstring.bz2 new file mode 100644 index 0000000..f5d5ce8 Binary files /dev/null and b/commit0/data/test_ids/bitstring.bz2 differ diff --git a/commit0/data/test_ids/bitstring.tar.bz2 b/commit0/data/test_ids/bitstring.tar.bz2 deleted file mode 100644 index b4470bc..0000000 Binary files a/commit0/data/test_ids/bitstring.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/cachetools.bz2 b/commit0/data/test_ids/cachetools.bz2 new file mode 100644 index 0000000..83a736a Binary files /dev/null and b/commit0/data/test_ids/cachetools.bz2 differ diff --git a/commit0/data/test_ids/cachetools.tar.bz2 b/commit0/data/test_ids/cachetools.tar.bz2 deleted file mode 100644 index 46a91ec..0000000 Binary files a/commit0/data/test_ids/cachetools.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/chardet.bz2 b/commit0/data/test_ids/chardet.bz2 new file mode 100644 index 0000000..cd316ad Binary files /dev/null and b/commit0/data/test_ids/chardet.bz2 differ diff --git a/commit0/data/test_ids/chardet.tar.bz2 b/commit0/data/test_ids/chardet.tar.bz2 deleted file mode 100644 index 8706e16..0000000 Binary files a/commit0/data/test_ids/chardet.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/click.bz2 b/commit0/data/test_ids/click.bz2 new file mode 100644 index 0000000..7e810b7 Binary files /dev/null and b/commit0/data/test_ids/click.bz2 differ diff --git a/commit0/data/test_ids/click.tar.bz2 b/commit0/data/test_ids/click.tar.bz2 deleted file mode 100644 index 7c81203..0000000 Binary files a/commit0/data/test_ids/click.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/cookiecutter.bz2 b/commit0/data/test_ids/cookiecutter.bz2 new file mode 100644 index 0000000..7d18be0 Binary files /dev/null and b/commit0/data/test_ids/cookiecutter.bz2 differ diff --git a/commit0/data/test_ids/cookiecutter.tar.bz2 b/commit0/data/test_ids/cookiecutter.tar.bz2 deleted file mode 100644 index 755419b..0000000 Binary files a/commit0/data/test_ids/cookiecutter.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/deprecated.bz2 b/commit0/data/test_ids/deprecated.bz2 new file mode 100644 index 0000000..72e1ee8 Binary files /dev/null and b/commit0/data/test_ids/deprecated.bz2 differ diff --git a/commit0/data/test_ids/deprecated.tar.bz2 b/commit0/data/test_ids/deprecated.tar.bz2 deleted file mode 100644 index 924e504..0000000 Binary files a/commit0/data/test_ids/deprecated.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/dnspython.bz2 b/commit0/data/test_ids/dnspython.bz2 new file mode 100644 index 0000000..86994ff Binary files /dev/null and b/commit0/data/test_ids/dnspython.bz2 differ diff --git a/commit0/data/test_ids/dnspython.tar.bz2 b/commit0/data/test_ids/dnspython.tar.bz2 deleted file mode 100644 index 461459c..0000000 Binary files a/commit0/data/test_ids/dnspython.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/dulwich.bz2 b/commit0/data/test_ids/dulwich.bz2 new file mode 100644 index 0000000..b2953ea Binary files /dev/null and b/commit0/data/test_ids/dulwich.bz2 differ diff --git a/commit0/data/test_ids/dulwich.tar.bz2 b/commit0/data/test_ids/dulwich.tar.bz2 deleted file mode 100644 index 896eb3d..0000000 Binary files a/commit0/data/test_ids/dulwich.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/fabric.bz2 b/commit0/data/test_ids/fabric.bz2 new file mode 100644 index 0000000..813399c Binary files /dev/null and b/commit0/data/test_ids/fabric.bz2 differ diff --git a/commit0/data/test_ids/fabric.tar.bz2 b/commit0/data/test_ids/fabric.tar.bz2 deleted file mode 100644 index 5d5711e..0000000 Binary files a/commit0/data/test_ids/fabric.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/fastapi.bz2 b/commit0/data/test_ids/fastapi.bz2 new file mode 100644 index 0000000..8375ad6 Binary files /dev/null and b/commit0/data/test_ids/fastapi.bz2 differ diff --git a/commit0/data/test_ids/fastapi.tar.bz2 b/commit0/data/test_ids/fastapi.tar.bz2 deleted file mode 100644 index dde15a3..0000000 Binary files a/commit0/data/test_ids/fastapi.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/filesystem_spec.bz2 b/commit0/data/test_ids/filesystem_spec.bz2 new file mode 100644 index 0000000..2472c51 Binary files /dev/null and b/commit0/data/test_ids/filesystem_spec.bz2 differ diff --git a/commit0/data/test_ids/filesystem_spec.tar.bz2 b/commit0/data/test_ids/filesystem_spec.tar.bz2 deleted file mode 100644 index 2125cff..0000000 Binary files a/commit0/data/test_ids/filesystem_spec.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/flask.bz2 b/commit0/data/test_ids/flask.bz2 new file mode 100644 index 0000000..4a6e7f4 Binary files /dev/null and b/commit0/data/test_ids/flask.bz2 differ diff --git a/commit0/data/test_ids/flask.tar.bz2 b/commit0/data/test_ids/flask.tar.bz2 deleted file mode 100644 index 82f5a02..0000000 Binary files a/commit0/data/test_ids/flask.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/geopandas.bz2 b/commit0/data/test_ids/geopandas.bz2 new file mode 100644 index 0000000..ae6eea2 Binary files /dev/null and b/commit0/data/test_ids/geopandas.bz2 differ diff --git a/commit0/data/test_ids/geopandas.tar.bz2 b/commit0/data/test_ids/geopandas.tar.bz2 deleted file mode 100644 index ab99e25..0000000 Binary files a/commit0/data/test_ids/geopandas.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/graphene.bz2 b/commit0/data/test_ids/graphene.bz2 new file mode 100644 index 0000000..21d29dc Binary files /dev/null and b/commit0/data/test_ids/graphene.bz2 differ diff --git a/commit0/data/test_ids/graphene.tar.bz2 b/commit0/data/test_ids/graphene.tar.bz2 deleted file mode 100644 index 9b5bd92..0000000 Binary files a/commit0/data/test_ids/graphene.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/imapclient.bz2 b/commit0/data/test_ids/imapclient.bz2 new file mode 100644 index 0000000..59558ba Binary files /dev/null and b/commit0/data/test_ids/imapclient.bz2 differ diff --git a/commit0/data/test_ids/imapclient.tar.bz2 b/commit0/data/test_ids/imapclient.tar.bz2 deleted file mode 100644 index 76158c8..0000000 Binary files a/commit0/data/test_ids/imapclient.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/imbalanced-learn.bz2 b/commit0/data/test_ids/imbalanced-learn.bz2 new file mode 100644 index 0000000..c9654a0 Binary files /dev/null and b/commit0/data/test_ids/imbalanced-learn.bz2 differ diff --git a/commit0/data/test_ids/imbalanced-learn.tar.bz2 b/commit0/data/test_ids/imbalanced-learn.tar.bz2 deleted file mode 100644 index b9bc6a7..0000000 Binary files a/commit0/data/test_ids/imbalanced-learn.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/jedi.bz2 b/commit0/data/test_ids/jedi.bz2 new file mode 100644 index 0000000..a445e01 Binary files /dev/null and b/commit0/data/test_ids/jedi.bz2 differ diff --git a/commit0/data/test_ids/jedi.tar.bz2 b/commit0/data/test_ids/jedi.tar.bz2 deleted file mode 100644 index c717395..0000000 Binary files a/commit0/data/test_ids/jedi.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/jinja.bz2 b/commit0/data/test_ids/jinja.bz2 new file mode 100644 index 0000000..0a02e6f Binary files /dev/null and b/commit0/data/test_ids/jinja.bz2 differ diff --git a/commit0/data/test_ids/jinja.tar.bz2 b/commit0/data/test_ids/jinja.tar.bz2 deleted file mode 100644 index 8be7cc2..0000000 Binary files a/commit0/data/test_ids/jinja.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/joblib.bz2 b/commit0/data/test_ids/joblib.bz2 new file mode 100644 index 0000000..754e5f6 Binary files /dev/null and b/commit0/data/test_ids/joblib.bz2 differ diff --git a/commit0/data/test_ids/joblib.tar.bz2 b/commit0/data/test_ids/joblib.tar.bz2 deleted file mode 100644 index 5409e31..0000000 Binary files a/commit0/data/test_ids/joblib.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/loguru.bz2 b/commit0/data/test_ids/loguru.bz2 new file mode 100644 index 0000000..4f7406b Binary files /dev/null and b/commit0/data/test_ids/loguru.bz2 differ diff --git a/commit0/data/test_ids/loguru.tar.bz2 b/commit0/data/test_ids/loguru.tar.bz2 deleted file mode 100644 index 18b0c0e..0000000 Binary files a/commit0/data/test_ids/loguru.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/marshmallow.bz2 b/commit0/data/test_ids/marshmallow.bz2 new file mode 100644 index 0000000..3b9ca31 Binary files /dev/null and b/commit0/data/test_ids/marshmallow.bz2 differ diff --git a/commit0/data/test_ids/marshmallow.tar.bz2 b/commit0/data/test_ids/marshmallow.tar.bz2 deleted file mode 100644 index 4e21f78..0000000 Binary files a/commit0/data/test_ids/marshmallow.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/mimesis.bz2 b/commit0/data/test_ids/mimesis.bz2 new file mode 100644 index 0000000..1b92e19 Binary files /dev/null and b/commit0/data/test_ids/mimesis.bz2 differ diff --git a/commit0/data/test_ids/mimesis.tar.bz2 b/commit0/data/test_ids/mimesis.tar.bz2 deleted file mode 100644 index cf9cd43..0000000 Binary files a/commit0/data/test_ids/mimesis.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/minitorch.bz2 b/commit0/data/test_ids/minitorch.bz2 new file mode 100644 index 0000000..b033bb1 Binary files /dev/null and b/commit0/data/test_ids/minitorch.bz2 differ diff --git a/commit0/data/test_ids/minitorch.tar.bz2 b/commit0/data/test_ids/minitorch.tar.bz2 deleted file mode 100644 index c4d9b99..0000000 Binary files a/commit0/data/test_ids/minitorch.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/more-itertools.bz2 b/commit0/data/test_ids/more-itertools.bz2 new file mode 100644 index 0000000..7bd805c Binary files /dev/null and b/commit0/data/test_ids/more-itertools.bz2 differ diff --git a/commit0/data/test_ids/more-itertools.tar.bz2 b/commit0/data/test_ids/more-itertools.tar.bz2 deleted file mode 100644 index 76f3625..0000000 Binary files a/commit0/data/test_ids/more-itertools.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/moviepy.bz2 b/commit0/data/test_ids/moviepy.bz2 new file mode 100644 index 0000000..e48c6c3 Binary files /dev/null and b/commit0/data/test_ids/moviepy.bz2 differ diff --git a/commit0/data/test_ids/moviepy.tar.bz2 b/commit0/data/test_ids/moviepy.tar.bz2 deleted file mode 100644 index 5194da7..0000000 Binary files a/commit0/data/test_ids/moviepy.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/networkx.bz2 b/commit0/data/test_ids/networkx.bz2 new file mode 100644 index 0000000..ddb1a50 Binary files /dev/null and b/commit0/data/test_ids/networkx.bz2 differ diff --git a/commit0/data/test_ids/networkx.tar.bz2 b/commit0/data/test_ids/networkx.tar.bz2 deleted file mode 100644 index 5ea11c9..0000000 Binary files a/commit0/data/test_ids/networkx.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/paramiko.bz2 b/commit0/data/test_ids/paramiko.bz2 new file mode 100644 index 0000000..068dfa6 Binary files /dev/null and b/commit0/data/test_ids/paramiko.bz2 differ diff --git a/commit0/data/test_ids/paramiko.tar.bz2 b/commit0/data/test_ids/paramiko.tar.bz2 deleted file mode 100644 index 044f7e1..0000000 Binary files a/commit0/data/test_ids/paramiko.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/parsel.bz2 b/commit0/data/test_ids/parsel.bz2 new file mode 100644 index 0000000..94c02a3 Binary files /dev/null and b/commit0/data/test_ids/parsel.bz2 differ diff --git a/commit0/data/test_ids/parsel.tar.bz2 b/commit0/data/test_ids/parsel.tar.bz2 deleted file mode 100644 index 5b2c03a..0000000 Binary files a/commit0/data/test_ids/parsel.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/pexpect.bz2 b/commit0/data/test_ids/pexpect.bz2 new file mode 100644 index 0000000..6ef7ec1 Binary files /dev/null and b/commit0/data/test_ids/pexpect.bz2 differ diff --git a/commit0/data/test_ids/pexpect.tar.bz2 b/commit0/data/test_ids/pexpect.tar.bz2 deleted file mode 100644 index 2252c7e..0000000 Binary files a/commit0/data/test_ids/pexpect.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/portalocker.bz2 b/commit0/data/test_ids/portalocker.bz2 new file mode 100644 index 0000000..2648ce9 Binary files /dev/null and b/commit0/data/test_ids/portalocker.bz2 differ diff --git a/commit0/data/test_ids/portalocker.tar.bz2 b/commit0/data/test_ids/portalocker.tar.bz2 deleted file mode 100644 index c416ed2..0000000 Binary files a/commit0/data/test_ids/portalocker.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/pyboy.bz2 b/commit0/data/test_ids/pyboy.bz2 new file mode 100644 index 0000000..dfbc441 Binary files /dev/null and b/commit0/data/test_ids/pyboy.bz2 differ diff --git a/commit0/data/test_ids/pyboy.tar.bz2 b/commit0/data/test_ids/pyboy.tar.bz2 deleted file mode 100644 index 0295906..0000000 Binary files a/commit0/data/test_ids/pyboy.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/pydantic.bz2 b/commit0/data/test_ids/pydantic.bz2 new file mode 100644 index 0000000..f8f08ca Binary files /dev/null and b/commit0/data/test_ids/pydantic.bz2 differ diff --git a/commit0/data/test_ids/pydantic.tar.bz2 b/commit0/data/test_ids/pydantic.tar.bz2 deleted file mode 100644 index 6b26abe..0000000 Binary files a/commit0/data/test_ids/pydantic.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/pyjwt.bz2 b/commit0/data/test_ids/pyjwt.bz2 new file mode 100644 index 0000000..596ac76 Binary files /dev/null and b/commit0/data/test_ids/pyjwt.bz2 differ diff --git a/commit0/data/test_ids/pyjwt.tar.bz2 b/commit0/data/test_ids/pyjwt.tar.bz2 deleted file mode 100644 index 961ae13..0000000 Binary files a/commit0/data/test_ids/pyjwt.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/pylint.bz2 b/commit0/data/test_ids/pylint.bz2 new file mode 100644 index 0000000..c427e03 Binary files /dev/null and b/commit0/data/test_ids/pylint.bz2 differ diff --git a/commit0/data/test_ids/pylint.tar.bz2 b/commit0/data/test_ids/pylint.tar.bz2 deleted file mode 100644 index 9f55cdb..0000000 Binary files a/commit0/data/test_ids/pylint.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/pypdf.bz2 b/commit0/data/test_ids/pypdf.bz2 new file mode 100644 index 0000000..b4f908b Binary files /dev/null and b/commit0/data/test_ids/pypdf.bz2 differ diff --git a/commit0/data/test_ids/pypdf.tar.bz2 b/commit0/data/test_ids/pypdf.tar.bz2 deleted file mode 100644 index 9824f39..0000000 Binary files a/commit0/data/test_ids/pypdf.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/pytest.bz2 b/commit0/data/test_ids/pytest.bz2 new file mode 100644 index 0000000..5e12232 Binary files /dev/null and b/commit0/data/test_ids/pytest.bz2 differ diff --git a/commit0/data/test_ids/pytest.tar.bz2 b/commit0/data/test_ids/pytest.tar.bz2 deleted file mode 100644 index 91b9553..0000000 Binary files a/commit0/data/test_ids/pytest.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/python-progressbar.bz2 b/commit0/data/test_ids/python-progressbar.bz2 new file mode 100644 index 0000000..609933a Binary files /dev/null and b/commit0/data/test_ids/python-progressbar.bz2 differ diff --git a/commit0/data/test_ids/python-progressbar.tar.bz2 b/commit0/data/test_ids/python-progressbar.tar.bz2 deleted file mode 100644 index 2b22872..0000000 Binary files a/commit0/data/test_ids/python-progressbar.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/python-prompt-toolkit.bz2 b/commit0/data/test_ids/python-prompt-toolkit.bz2 new file mode 100644 index 0000000..6051d29 Binary files /dev/null and b/commit0/data/test_ids/python-prompt-toolkit.bz2 differ diff --git a/commit0/data/test_ids/python-prompt-toolkit.tar.bz2 b/commit0/data/test_ids/python-prompt-toolkit.tar.bz2 deleted file mode 100644 index 461428d..0000000 Binary files a/commit0/data/test_ids/python-prompt-toolkit.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/python-rsa.bz2 b/commit0/data/test_ids/python-rsa.bz2 new file mode 100644 index 0000000..d4ffeab Binary files /dev/null and b/commit0/data/test_ids/python-rsa.bz2 differ diff --git a/commit0/data/test_ids/python-rsa.tar.bz2 b/commit0/data/test_ids/python-rsa.tar.bz2 deleted file mode 100644 index 2e76490..0000000 Binary files a/commit0/data/test_ids/python-rsa.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/requests.bz2 b/commit0/data/test_ids/requests.bz2 new file mode 100644 index 0000000..e60041d Binary files /dev/null and b/commit0/data/test_ids/requests.bz2 differ diff --git a/commit0/data/test_ids/requests.tar.bz2 b/commit0/data/test_ids/requests.tar.bz2 deleted file mode 100644 index a5dd6e2..0000000 Binary files a/commit0/data/test_ids/requests.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/scrapy.bz2 b/commit0/data/test_ids/scrapy.bz2 new file mode 100644 index 0000000..ac31c03 Binary files /dev/null and b/commit0/data/test_ids/scrapy.bz2 differ diff --git a/commit0/data/test_ids/scrapy.tar.bz2 b/commit0/data/test_ids/scrapy.tar.bz2 deleted file mode 100644 index 85a5f3d..0000000 Binary files a/commit0/data/test_ids/scrapy.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/seaborn.bz2 b/commit0/data/test_ids/seaborn.bz2 new file mode 100644 index 0000000..83e7c20 Binary files /dev/null and b/commit0/data/test_ids/seaborn.bz2 differ diff --git a/commit0/data/test_ids/seaborn.tar.bz2 b/commit0/data/test_ids/seaborn.tar.bz2 deleted file mode 100644 index 934bc62..0000000 Binary files a/commit0/data/test_ids/seaborn.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/simpy.bz2 b/commit0/data/test_ids/simpy.bz2 new file mode 100644 index 0000000..227d282 Binary files /dev/null and b/commit0/data/test_ids/simpy.bz2 differ diff --git a/commit0/data/test_ids/simpy.tar.bz2 b/commit0/data/test_ids/simpy.tar.bz2 deleted file mode 100644 index 957f263..0000000 Binary files a/commit0/data/test_ids/simpy.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/sphinx.bz2 b/commit0/data/test_ids/sphinx.bz2 new file mode 100644 index 0000000..b8aba35 Binary files /dev/null and b/commit0/data/test_ids/sphinx.bz2 differ diff --git a/commit0/data/test_ids/sphinx.tar.bz2 b/commit0/data/test_ids/sphinx.tar.bz2 deleted file mode 100644 index 37ddb52..0000000 Binary files a/commit0/data/test_ids/sphinx.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/sqlparse.bz2 b/commit0/data/test_ids/sqlparse.bz2 new file mode 100644 index 0000000..96e0e6d Binary files /dev/null and b/commit0/data/test_ids/sqlparse.bz2 differ diff --git a/commit0/data/test_ids/sqlparse.tar.bz2 b/commit0/data/test_ids/sqlparse.tar.bz2 deleted file mode 100644 index 3ea4bc1..0000000 Binary files a/commit0/data/test_ids/sqlparse.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/statsmodels.bz2 b/commit0/data/test_ids/statsmodels.bz2 new file mode 100644 index 0000000..e937c98 Binary files /dev/null and b/commit0/data/test_ids/statsmodels.bz2 differ diff --git a/commit0/data/test_ids/statsmodels.tar.bz2 b/commit0/data/test_ids/statsmodels.tar.bz2 deleted file mode 100644 index 796e69e..0000000 Binary files a/commit0/data/test_ids/statsmodels.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/tinydb.bz2 b/commit0/data/test_ids/tinydb.bz2 new file mode 100644 index 0000000..3df304e Binary files /dev/null and b/commit0/data/test_ids/tinydb.bz2 differ diff --git a/commit0/data/test_ids/tinydb.tar.bz2 b/commit0/data/test_ids/tinydb.tar.bz2 deleted file mode 100644 index 663d600..0000000 Binary files a/commit0/data/test_ids/tinydb.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/tlslite-ng.bz2 b/commit0/data/test_ids/tlslite-ng.bz2 new file mode 100644 index 0000000..16d2732 Binary files /dev/null and b/commit0/data/test_ids/tlslite-ng.bz2 differ diff --git a/commit0/data/test_ids/tlslite-ng.tar.bz2 b/commit0/data/test_ids/tlslite-ng.tar.bz2 deleted file mode 100644 index e86a904..0000000 Binary files a/commit0/data/test_ids/tlslite-ng.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/tornado.bz2 b/commit0/data/test_ids/tornado.bz2 new file mode 100644 index 0000000..774ddeb Binary files /dev/null and b/commit0/data/test_ids/tornado.bz2 differ diff --git a/commit0/data/test_ids/tornado.tar.bz2 b/commit0/data/test_ids/tornado.tar.bz2 deleted file mode 100644 index a0e10f2..0000000 Binary files a/commit0/data/test_ids/tornado.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/virtualenv.bz2 b/commit0/data/test_ids/virtualenv.bz2 new file mode 100644 index 0000000..298327b Binary files /dev/null and b/commit0/data/test_ids/virtualenv.bz2 differ diff --git a/commit0/data/test_ids/virtualenv.tar.bz2 b/commit0/data/test_ids/virtualenv.tar.bz2 deleted file mode 100644 index 74c5cc5..0000000 Binary files a/commit0/data/test_ids/virtualenv.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/voluptuous.bz2 b/commit0/data/test_ids/voluptuous.bz2 new file mode 100644 index 0000000..8fbdc62 Binary files /dev/null and b/commit0/data/test_ids/voluptuous.bz2 differ diff --git a/commit0/data/test_ids/voluptuous.tar.bz2 b/commit0/data/test_ids/voluptuous.tar.bz2 deleted file mode 100644 index 55d93c1..0000000 Binary files a/commit0/data/test_ids/voluptuous.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/wcwidth.bz2 b/commit0/data/test_ids/wcwidth.bz2 new file mode 100644 index 0000000..50af469 Binary files /dev/null and b/commit0/data/test_ids/wcwidth.bz2 differ diff --git a/commit0/data/test_ids/wcwidth.tar.bz2 b/commit0/data/test_ids/wcwidth.tar.bz2 deleted file mode 100644 index c51eb40..0000000 Binary files a/commit0/data/test_ids/wcwidth.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/web3-py.bz2 b/commit0/data/test_ids/web3-py.bz2 new file mode 100644 index 0000000..ffa3dc8 Binary files /dev/null and b/commit0/data/test_ids/web3-py.bz2 differ diff --git a/commit0/data/test_ids/web3-py.tar.bz2 b/commit0/data/test_ids/web3-py.tar.bz2 deleted file mode 100644 index 2fa654c..0000000 Binary files a/commit0/data/test_ids/web3-py.tar.bz2 and /dev/null differ diff --git a/commit0/data/test_ids/xarray.bz2 b/commit0/data/test_ids/xarray.bz2 new file mode 100644 index 0000000..5e56619 Binary files /dev/null and b/commit0/data/test_ids/xarray.bz2 differ diff --git a/commit0/data/test_ids/xarray.tar.bz2 b/commit0/data/test_ids/xarray.tar.bz2 deleted file mode 100644 index 3f38b5c..0000000 Binary files a/commit0/data/test_ids/xarray.tar.bz2 and /dev/null differ diff --git a/commit0/harness/constants.py b/commit0/harness/constants.py index 9a92337..19d13a1 100644 --- a/commit0/harness/constants.py +++ b/commit0/harness/constants.py @@ -9,6 +9,7 @@ class RepoInstance(TypedDict): reference_commit: str setup: dict test: Dict[str, str] + src_dir: str class Files(TypedDict): diff --git a/commit0/harness/evaluate.py b/commit0/harness/evaluate.py index ddc4b15..5b6b721 100644 --- a/commit0/harness/evaluate.py +++ b/commit0/harness/evaluate.py @@ -24,6 +24,7 @@ def main( repo_split: str, base_dir: str, branch: Union[str, None], + coverage: bool, backend: str, timeout: int, num_cpus: int, @@ -58,6 +59,7 @@ def main( repo, branch, test_dir, + coverage, backend, timeout, num_cpus, diff --git a/commit0/harness/execution_context.py b/commit0/harness/execution_context.py index 2cd79ba..fa9e3b8 100644 --- a/commit0/harness/execution_context.py +++ b/commit0/harness/execution_context.py @@ -163,7 +163,7 @@ def __init__( # the image must exist on dockerhub reponame = spec.repo.split("/")[-1] - image_name = f"wentingzhao/{reponame}:latest".lower() + image_name = f"wentingzhao/{reponame}:v0".lower() image = modal.Image.from_registry(image_name, force_build=rebuild_image) if files_to_copy: for _, f in files_to_copy.items(): diff --git a/commit0/harness/get_pytest_ids.py b/commit0/harness/get_pytest_ids.py index 36ce65a..5ed2ce7 100644 --- a/commit0/harness/get_pytest_ids.py +++ b/commit0/harness/get_pytest_ids.py @@ -1,20 +1,14 @@ -import tarfile +import bz2 from typing import List def main(repo: str, verbose: int) -> List[str]: repo = repo.lower() repo = repo.replace(".", "-") - out = "" - with tarfile.open(f"commit0/data/test_ids/{repo}.tar.bz2", "r:bz2") as tar: - for member in tar.getmembers(): - if member.isfile(): - file = tar.extractfile(member) - if file: - content = file.read().decode("utf-8") - out += content - if verbose: - print(content) + with bz2.open(f"commit0/data/test_ids/{repo}.bz2", "rt") as f: + out = f.read() + if verbose: + print(out) out = out.split("\n") return out diff --git a/commit0/harness/lint.py b/commit0/harness/lint.py index ff2bd34..d073fd5 100644 --- a/commit0/harness/lint.py +++ b/commit0/harness/lint.py @@ -1,7 +1,13 @@ import subprocess import sys +import os +from datasets import load_dataset from pathlib import Path -from typing import List +from typing import Iterator + +from commit0.harness.constants import ( + RepoInstance, +) config = """repos: @@ -28,7 +34,38 @@ - id: pyright""" -def main(files: List[Path]) -> None: +def main( + dataset_name: str, dataset_split: str, repo_or_repo_dir: str, base_dir: str +) -> None: + dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split) # type: ignore + example = None + repo_name = None + for example in dataset: + repo_name = example["repo"].split("/")[-1] + if repo_or_repo_dir.endswith("/"): + repo_or_repo_dir = repo_or_repo_dir[:-1] + if repo_name in os.path.basename(repo_or_repo_dir): + break + assert example is not None, "No example available" + assert repo_name is not None, "No repo available" + + repo_dir = os.path.join(base_dir, repo_name) + if os.path.isdir(repo_or_repo_dir): + repo = repo_or_repo_dir + elif os.path.isdir(repo_dir): + repo = repo_dir + else: + raise Exception( + f"Neither {repo_dir} nor {repo_or_repo_dir} is a valid path.\nUsage: commit0 lint {{repo_or_repo_dir}}" + ) + + files = [] + repo = os.path.join(repo, example["src_dir"]) + for root, dirs, fs in os.walk(repo): + for file in fs: + if file.endswith(".py"): + files.append(os.path.join(root, file)) + config_file = Path(".commit0.pre-commit-config.yaml") if not config_file.is_file(): config_file.write_text(config) diff --git a/commit0/harness/run_pytest_ids.py b/commit0/harness/run_pytest_ids.py index fd45080..0863796 100644 --- a/commit0/harness/run_pytest_ids.py +++ b/commit0/harness/run_pytest_ids.py @@ -34,6 +34,7 @@ def main( repo_or_repo_dir: str, branch: str, test_ids: str, + coverage: bool, backend: str, timeout: int, num_cpus: int, @@ -113,7 +114,11 @@ def main( patch_file.write_text(patch) # make eval file - eval_script = spec.eval_script.format(test_ids=test_ids) + if coverage: + coverage_text = f" --cov={example['src_dir']} --cov-branch --cov-report json" + else: + coverage_text = "" + eval_script = spec.eval_script.format(test_ids=test_ids, coverage=coverage_text) eval_file = Path(log_dir / "eval.sh") eval_file.write_text(eval_script) @@ -134,10 +139,11 @@ def main( ) files_to_collect = [ "report.json", - "coverage.json", "pytest_exit_code.txt", "test_output.txt", ] + if coverage: + files_to_collect.append("coverage.json") try: with execution_context( diff --git a/commit0/harness/spec.py b/commit0/harness/spec.py index fc04355..dc620ef 100644 --- a/commit0/harness/spec.py +++ b/commit0/harness/spec.py @@ -54,13 +54,13 @@ def repo_image_key(self) -> str: repo = self.repo.split("/")[-1] # this is the image name created locally # once this image created, it will be tagged with repo_image_tag - return f"commit0.repo.{repo}.{val}:latest".lower() + return f"commit0.repo.{repo}.{val}:v0".lower() @property def repo_image_tag(self) -> str: """Repo image tag that will be used throughout.""" repo = self.repo.split("/")[-1] - return f"wentingzhao/{repo}:latest".lower() + return f"wentingzhao/{repo}:v0".lower() def get_container_name(self, run_id: Optional[str] = None) -> str: repo = self.repo.split("/")[-1] @@ -146,7 +146,7 @@ def make_repo_script_list(instance: RepoInstance, repo_directory: str) -> list[s ) setup_commands.append(install) setup_commands.append( - "uv pip install pytest pytest-cov coverage pytest-json-report" + "uv pip install -U pytest pytest-cov coverage pytest-json-report" ) setup_commands.append(f"git reset --hard {base_commit}") return setup_commands @@ -160,10 +160,8 @@ def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[s f"git reset --hard {instance['base_commit']}", "git apply --allow-empty -v /patch.diff", "git status", - f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors --cov=. --cov-branch --cov-report json {{test_ids}} > test_output.txt 2>&1", + f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json --continue-on-collection-errors{{coverage}} {{test_ids}} > test_output.txt 2>&1", "echo $? > pytest_exit_code.txt", - f"git reset --hard {instance['base_commit']}", - "git status", ] return eval_script_list