diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml index 5d52534..cbc05e4 100644 --- a/.github/workflows/pr-evaluation.yml +++ b/.github/workflows/pr-evaluation.yml @@ -1,29 +1,62 @@ name: Router Submission Evaluation on: - pull_request_target: - types: [opened, synchronize, reopened] - paths: - - "router_inference/predictions/**" + issue_comment: + types: [created] jobs: evaluate-router: + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/evaluate') && + ( + github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'COLLABORATOR' || + github.event.comment.user.login == github.event.issue.user.login + ) runs-on: self-hosted permissions: contents: read pull-requests: write steps: + - name: Acknowledge /evaluate command + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes' + }); + + - name: Fetch PR details + id: pr + uses: actions/github-script@v7 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number + }); + core.setOutput('head_sha', pr.data.head.sha); + core.setOutput('base_ref', pr.data.base.ref); + core.setOutput('base_sha', pr.data.base.sha); + core.setOutput('number', pr.data.number); + - name: Checkout base repository (for evaluation scripts) uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.base.ref }} + ref: ${{ steps.pr.outputs.base_ref }} path: base fetch-depth: 0 - name: Checkout PR branch (for prediction file only) uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ steps.pr.outputs.head_sha }} path: pr fetch-depth: 0 @@ -35,8 +68,8 @@ jobs: set -euo pipefail # Compare against the upstream base branch # This ensures each router submission is evaluated independently - BASE_REF="${{ github.event.pull_request.base.ref }}" - BASE_SHA="${{ github.event.pull_request.base.sha }}" + BASE_REF="${{ steps.pr.outputs.base_ref }}" + BASE_SHA="${{ steps.pr.outputs.base_sha }}" if [[ -z "$BASE_SHA" ]]; then echo "Error: Could not determine PR base SHA" >&2 @@ -158,9 +191,9 @@ jobs: run: | set -euo pipefail; trap 'cat evaluation_output.txt' EXIT # Uses base repo's evaluation script (safe - not from PR) - BASE_SHA="${{ github.event.pull_request.base.sha }}" + BASE_SHA="${{ steps.pr.outputs.base_sha }}" uv run python automation/process_pr_submission.py \ - --pr "${{ github.event.pull_request.number }}" \ + --pr "${{ steps.pr.outputs.number }}" \ --router "${{ steps.detect.outputs.router }}" \ --split "${{ steps.detect.outputs.split }}" \ --base-ref "$BASE_SHA" > evaluation_output.txt 2>&1 @@ -213,9 +246,21 @@ jobs: // Post comment to PR await github.rest.issues.createComment({ - issue_number: context.payload.pull_request.number, + issue_number: context.payload.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: comment }); console.log('Successfully posted evaluation results as PR comment'); + + - name: React with success + if: ${{ steps.detect.outputs.router != '' && steps.evaluate.outcome == 'success' }} + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'rocket' + });