diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df9148c4b95b..8d44f2a4b2a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ concurrency: jobs: lint: # eliminate duplicate runs on master - if: github.event_name == 'push' || github.ref_name != 'master' || (github.event.pull_request.head.repo.fork == (github.event_name == 'pull_request_target')) + if: github.event_name == 'push' || github.base_ref != 'master' || (github.event.pull_request.head.repo.fork == (github.event_name == 'pull_request_target')) runs-on: ubuntu-latest env: @@ -37,14 +37,15 @@ jobs: with: python-version: ${{ env.min-python-version }} - - name: Set Date - run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + - id: get_date + name: Get date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - name: Cache Python packages + - name: Set up Python dependency cache uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ env.DATE }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }} - name: Install dependencies run: | @@ -73,7 +74,7 @@ jobs: test: # eliminate duplicate runs on master - if: github.event_name == 'push' || github.ref_name != 'master' || (github.event.pull_request.head.repo.fork == (github.event_name == 'pull_request_target')) + if: github.event_name == 'push' || github.base_ref != 'master' || (github.event.pull_request.head.repo.fork == (github.event_name == 'pull_request_target')) permissions: # Gives the action the necessary permissions for publishing new @@ -90,7 +91,7 @@ jobs: python-version: ["3.10"] steps: - - name: Check out repository + - name: Checkout repository uses: actions/checkout@v3 with: fetch-depth: 0 @@ -98,8 +99,12 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name }} submodules: true - - id: checkout_cassettes - name: Check out cassettes + - name: Configure git user Auto-GPT-Bot + run: | + git config --global user.name "Auto-GPT-Bot" + git config --global user.email "github-bot@agpt.co" + + - name: Checkout cassettes if: ${{ startsWith(github.event_name, 'pull_request') }} run: | cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" @@ -111,21 +116,14 @@ jobs: git checkout $cassette_branch - if git merge --no-commit --no-ff ${{ github.event.pull_request.base.ref }}; then - echo "Using cassettes from mirror branch, synced to upstream branch '${{ github.event.pull_request.base.ref }}'" - else - echo "Could not merge upstream changes to cassettes. Using cassettes from ${{ github.event.pull_request.base.ref }}." - git merge --abort - git checkout ${{ github.event.pull_request.base.ref }} - - # Delete branch to prevent conflict when re-creating it - git branch -D $cassette_branch - fi - echo "cassette_branch=$(git branch --show-current)" >> $GITHUB_OUTPUT + # Pick non-conflicting cassette updates from the base branch + git merge --no-commit --strategy-option=ours origin/${{ github.event.pull_request.base.ref }} + echo "Using cassettes from mirror branch '$cassette_branch'," \ + "synced to upstream branch '${{ github.event.pull_request.base.ref }}'." else - echo "Branch '$cassette_branch' does not exist in cassette submodule."\ - "Using cassettes from ${{ github.event.pull_request.base.ref }}." - echo "cassette_branch=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT + git checkout -b $cassette_branch + echo "Branch '$cassette_branch' does not exist in cassette submodule." \ + "Using cassettes from '${{ github.event.pull_request.base.ref }}'." fi - name: Set up Python ${{ matrix.python-version }} @@ -133,21 +131,22 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Set Date - run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + - id: get_date + name: Get date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - name: Cache Python packages + - name: Set up Python dependency cache uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ env.DATE }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }} - - name: Install dependencies + - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - - name: Run pytest tests with coverage + - name: Run pytest with coverage run: | pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term python tests/integration/challenges/utils/build_current_score.py @@ -162,10 +161,9 @@ jobs: - id: setup_git_auth name: Set up git token authentication + # Cassettes may be pushed even when tests fail + if: success() || failure() run: | - git config --global user.name "Auto-GPT-Bot" - git config --global user.email "github-bot@agpt.co" - config_key="http.${{ github.server_url }}/.extraheader" base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0) @@ -186,58 +184,39 @@ jobs: if ! git diff --quiet $score_file; then git add $score_file git commit -m "Update challenge scores" - git push origin HEAD:${{ github.ref }} + git push origin HEAD:${{ github.ref_name }} else echo "The challenge scores didn't change." fi - id: push_cassettes name: Push updated cassettes + # For pull requests, push updated cassettes even when tests fail + if: github.event_name == 'push' || success() || failure() run: | if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then is_pull_request=true cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" - cassette_source_branch="${{ steps.checkout_cassettes.outputs.cassette_branch }}" - base_branch="${{ github.event.pull_request.base.ref }}" else - current_branch=$(echo ${{ github.ref }} | sed -e "s/refs\/heads\///g") - cassette_branch=$current_branch + cassette_branch="${{ github.ref_name }}" fi cd tests/Auto-GPT-test-cassettes - git fetch origin $cassette_source_branch:$cassette_source_branch - # Commit & push changes to cassettes if any - if ! git diff --quiet $cassette_source_branch --; then - if [ "$cassette_branch" != "$cassette_source_branch" ]; then - git checkout -b $cassette_branch - fi + if ! git diff --quiet; then git add . git commit -m "Auto-update cassettes" - - if [ $is_pull_request ]; then - git push --force origin HEAD:$cassette_branch - else - git push origin HEAD:$cassette_branch - fi - - cd ../.. - if [ $is_pull_request ]; then - git fetch origin $base_branch - cassette_diff=$(git diff origin/$base_branch) - else + git push origin HEAD:$cassette_branch + if [ ! $is_pull_request ]; then + cd ../.. git add tests/Auto-GPT-test-cassettes git commit -m "Update cassette submodule" - git push origin HEAD:$current_branch + git push origin HEAD:$cassette_branch fi - else - echo "No cassette changes to commit" - fi - - if [ -n "$cassette_diff" ]; then echo "updated=true" >> $GITHUB_OUTPUT else echo "updated=false" >> $GITHUB_OUTPUT + echo "No cassette changes to commit" fi - name: Post Set up git token auth @@ -246,7 +225,7 @@ jobs: git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' - - name: Apply or remove behaviour change label and comment on PR + - name: Apply "behaviour change" label and comment on PR if: ${{ startsWith(github.event_name, 'pull_request') }} run: | PR_NUMBER=${{ github.event.pull_request.number }} @@ -263,10 +242,4 @@ jobs: echo $TOKEN | gh auth login --with-token gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged." - else - echo "Removing label..." - curl -X DELETE \ - -H "Authorization: Bearer $TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/behaviour%20change fi diff --git a/tests/integration/challenges/basic_abilities/test_write_file.py b/tests/integration/challenges/basic_abilities/test_write_file.py index cbbad514b6b6..393dbfd05d85 100644 --- a/tests/integration/challenges/basic_abilities/test_write_file.py +++ b/tests/integration/challenges/basic_abilities/test_write_file.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -17,7 +18,7 @@ @challenge def test_write_file( writer_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py index 6b970e8b227d..2f61fef3902a 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py +++ b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.commands.file_operations import read_file from autogpt.config import Config @@ -19,7 +20,7 @@ def test_information_retrieval_challenge_a( information_retrieval_agents: Agent, monkeypatch: pytest.MonkeyPatch, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, config: Config, level_to_run: int, ) -> None: diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py index feac95a0f646..6461e13fbf4c 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py +++ b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py @@ -1,6 +1,7 @@ import contextlib import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -20,7 +21,7 @@ def test_information_retrieval_challenge_b( get_nobel_prize_agent: Agent, monkeypatch: pytest.MonkeyPatch, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, level_to_run: int, config: Config, ) -> None: diff --git a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py b/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py index 5fd280ac4bbb..aa46ac4d4f33 100644 --- a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py +++ b/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py @@ -1,5 +1,6 @@ import pytest import yaml +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -19,6 +20,7 @@ def test_kubernetes_template_challenge_a( kubernetes_agent: Agent, monkeypatch: pytest.MonkeyPatch, + patched_api_requestor: MockerFixture, config: Config, level_to_run: int, ) -> None: diff --git a/tests/integration/challenges/memory/test_memory_challenge_a.py b/tests/integration/challenges/memory/test_memory_challenge_a.py index 8919bf58c0eb..08f461bdd86d 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_a.py +++ b/tests/integration/challenges/memory/test_memory_challenge_a.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -15,7 +16,7 @@ @challenge def test_memory_challenge_a( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/memory/test_memory_challenge_b.py b/tests/integration/challenges/memory/test_memory_challenge_b.py index 5c28b330a3b4..c82763129aca 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_b.py +++ b/tests/integration/challenges/memory/test_memory_challenge_b.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -17,7 +18,7 @@ @challenge def test_memory_challenge_b( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/memory/test_memory_challenge_c.py b/tests/integration/challenges/memory/test_memory_challenge_c.py index 23c0217d6135..ab8ece105262 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_c.py +++ b/tests/integration/challenges/memory/test_memory_challenge_c.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -18,7 +19,7 @@ @challenge def test_memory_challenge_c( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int,