diff --git a/.github/actions/bencher-track/action.yml b/.github/actions/bencher-track/action.yml index d37d179d..498f19af 100644 --- a/.github/actions/bencher-track/action.yml +++ b/.github/actions/bencher-track/action.yml @@ -1,10 +1,10 @@ name: Track benchmarks on bencher description: >- Run `bencher run` for the `ix` project. Every measure's baseline is windowed - to the data points recorded since the workload's `baseline-reset-` + to the data points recorded since the workload's `bencher-thresholds-reset-` tag, so a toolchain bump (or any permanent shift) is recalibrated by moving that one tag — via the !bencher-thresholds-reset PR comment or the - bench-thresholds-reset workflow. The calling job must check out with + bencher-thresholds-reset workflow. The calling job must check out with fetch-depth: 0 and fetch-tags: true (the tag is read here). inputs: @@ -12,7 +12,7 @@ inputs: description: Bencher testbed slug. required: true workload: - description: Workload key for the `baseline-reset-` tag (e.g. ix-compile, aiur). + description: Workload key for the `bencher-thresholds-reset-` tag (e.g. ix-compile, aiur). required: true file: description: Bencher Metric Format JSON file to upload. @@ -45,7 +45,7 @@ runs: THRESHOLDS: ${{ inputs.thresholds }} run: | # Baseline window: data points for this benchmark since the workload's - # `baseline-reset-` tag (moved on a toolchain bump or any + # `bencher-thresholds-reset-` tag (moved on a toolchain bump or any # permanent shift). The same window applies to every measure. We count # actual data points, not commits, so the baseline never reaches past # the anchor (a failed run produces no metric, so a commit count would @@ -54,7 +54,7 @@ runs: # project is public, so no token is needed. With no tag yet, fall back # to Bencher's 64-point window. sample=64 - if reset_epoch=$(git log -1 --format=%ct "baseline-reset-$WORKLOAD" 2>/dev/null); then + if reset_epoch=$(git log -1 --format=%ct "bencher-thresholds-reset-$WORKLOAD" 2>/dev/null); then bench=$(jq -r '[keys[]][0]' "$FILE") sample=$(curl -fsS "https://api.bencher.dev/v0/projects/ix/reports?branch=${GITHUB_REF_NAME}&testbed=${TESTBED}&start_time=$((reset_epoch * 1000))&per_page=255" \ | jq --arg b "$bench" '[.[] | select([.results[]?[]?.benchmark.name] | index($b))] | length') diff --git a/.github/workflows/bench-main.yml b/.github/workflows/bench-main.yml index 9b645594..957b51b3 100644 --- a/.github/workflows/bench-main.yml +++ b/.github/workflows/bench-main.yml @@ -81,7 +81,7 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - fetch-tags: true # bencher-track reads the baseline-reset tag + fetch-tags: true # bencher-track reads the bencher-thresholds-reset tag - uses: actions/cache/restore@v5 with: path: ~/.local/bin @@ -182,7 +182,7 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 # full history for the baseline-anchor lookup - fetch-tags: true # bencher-track reads the baseline-reset tag + fetch-tags: true # bencher-track reads the bencher-thresholds-reset tag - uses: actions/cache/restore@v5 with: path: ~/.local/bin diff --git a/.github/workflows/bench-thresholds-reset.yml b/.github/workflows/bench-thresholds-reset.yml deleted file mode 100644 index 4f17e558..00000000 --- a/.github/workflows/bench-thresholds-reset.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Bencher thresholds reset - -# Move `baseline-reset-` tag(s) so bencher-track re-windows those -# measures from a chosen commit — after a toolchain bump or any permanent shift. -# Other workloads are unaffected; historical data stays in Bencher, only the -# alert baseline is re-anchored. Two ways to trigger: -# -# - Merged PR carrying a maintainer comment `!bencher-thresholds-reset `: -# fires automatically on merge, anchored at the merge commit — no need to -# race a manual run afterward. A bare command (no workload) is ignored. -# - workflow_dispatch: reset the chosen workload at a given commit (default -# HEAD); does not read PR contents. For ad-hoc or bootstrap resets. -# -# pull_request_target runs in the base-repo context with a write token (works -# for fork PRs too). Safe here because the job runs no PR-provided code — only -# `gh api` calls against the trusted event payload. -on: - workflow_dispatch: - inputs: - workload: - description: Workload baseline to reset - required: true - type: choice - options: [ix-compile, aiur, all] - sha: - description: "Commit to anchor to (default: HEAD)" - required: false - pull_request_target: - types: [closed] - -permissions: - contents: write - pull-requests: write - -jobs: - reset: - if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true - runs-on: ubuntu-latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - EVENT: ${{ github.event_name }} - INPUT_WORKLOAD: ${{ inputs.workload }} - INPUT_SHA: ${{ inputs.sha }} - HEAD_SHA: ${{ github.sha }} - PR: ${{ github.event.pull_request.number }} - MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} - steps: - - run: | - valid="ix-compile aiur" - if [ "$EVENT" = workflow_dispatch ]; then - # Reset the chosen workload(s) at the given commit; no PR scan. - sha="${INPUT_SHA:-$HEAD_SHA}" - [ "$INPUT_WORKLOAD" = all ] && workloads="$valid" || workloads="$INPUT_WORKLOAD" - else - # Reset workloads named in maintainer `!bencher-thresholds-reset` - # comments, anchored at the merge commit. Arg required, so a bare - # command matches nothing and is ignored. - sha="$MERGE_SHA" - workloads=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \ - --jq '.[] | select(.author_association | IN("OWNER","MEMBER","COLLABORATOR")) | .body' \ - | grep -oiE '!bencher-thresholds-reset[[:space:]]+[a-z0-9 -]+' \ - | sed -E 's/^!bencher-thresholds-reset[[:space:]]+//I' \ - | tr '[:upper:] ' '[:lower:]\n' | sort -u) - echo "$workloads" | grep -qx all && workloads="$valid" - fi - - done_list="" - for w in $workloads; do - case " $valid " in *" $w "*) ;; *) echo "skip unknown workload: $w"; continue ;; esac - tag="baseline-reset-$w" - echo "Anchoring $tag -> $sha" - # Update the tag if it exists, else create it. (Checking first avoids - # the spurious 422 a PATCH-then-POST logs on first creation.) - if gh api --silent "repos/$REPO/git/refs/tags/$tag" 2>/dev/null; then - gh api --silent -X PATCH "repos/$REPO/git/refs/tags/$tag" -f sha="$sha" -F force=true - else - gh api --silent -X POST "repos/$REPO/git/refs" -f ref="refs/tags/$tag" -f sha="$sha" - fi - done_list="$done_list $w" - done - [ -z "$done_list" ] && { echo "Nothing to reset."; exit 0; } - - # Confirm on the PR for the merge-triggered path. - [ "$EVENT" = pull_request_target ] && gh pr comment "$PR" --repo "$REPO" \ - --body "♻️ Baseline reset to \`${sha:0:7}\` for:$done_list" - true diff --git a/.github/workflows/bencher-thresholds-reset.yml b/.github/workflows/bencher-thresholds-reset.yml new file mode 100644 index 00000000..37a0213c --- /dev/null +++ b/.github/workflows/bencher-thresholds-reset.yml @@ -0,0 +1,146 @@ +name: Bencher thresholds reset + +# Move `bencher-thresholds-reset-` tag(s) so bencher-track re-windows those +# measures from a chosen commit — after a toolchain bump or any permanent shift. +# Other workloads are unaffected; historical data stays in Bencher, only the +# alert baseline is re-anchored. Two ways to trigger: +# +# - Maintainer comment `!bencher-thresholds-reset ` on a PR: +# the `ack` job (issue_comment) is the single place that parses/validates the +# command. It records each valid workload as a `bencher-thresholds-reset:` +# PR label and replies to confirm (or warns on an unknown/missing workload, +# catching a typo immediately). On merge, the `reset` job simply reads those +# labels from the event payload — no re-parsing — and anchors the tags at the +# merge commit. The label is the shared flag between the two events, and is +# human-visible/cancelable (remove the label to cancel a queued reset). +# - Manual label (no comment needed): the merge step honors any +# `bencher-thresholds-reset:` label on the PR, whatever added it — so a +# Triage+ collaborator can queue a reset by applying the label directly, and +# cancel by removing it before merge. Naming convention: one label per token, +# `bencher-thresholds-reset:` where is `ix-compile`, `aiur`, or +# `all` (the merge step expands an `all` label into every workload). Labeling +# requires Triage+, so PR authors from forks cannot self-queue a reset. The +# label shares the command/workflow name; the git tag it moves is the same +# stem with a dash: `bencher-thresholds-reset-`. +# - workflow_dispatch: reset the chosen workload at a given commit (default +# HEAD); does not read PR contents. For ad-hoc or bootstrap resets. +# +# pull_request_target / issue_comment both run in the base-repo context with a +# write token (works for fork PRs too). Safe here because the jobs run no +# PR-provided code — only `gh` calls against the trusted event payload, with the +# comment body used purely as data (never evaluated). +on: + workflow_dispatch: + inputs: + workload: + description: Workload baseline to reset + required: true + type: choice + options: [ix-compile, aiur, all] + sha: + description: "Commit to anchor to (default: HEAD)" + required: false + pull_request_target: + types: [closed] + issue_comment: + types: [created] + +permissions: + contents: write + pull-requests: write + issues: write # ack creates/applies the bencher-thresholds-reset: label + +jobs: + reset: + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + EVENT: ${{ github.event_name }} + INPUT_WORKLOAD: ${{ inputs.workload }} + INPUT_SHA: ${{ inputs.sha }} + HEAD_SHA: ${{ github.sha }} + PR: ${{ github.event.pull_request.number }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + steps: + - run: | + valid="ix-compile aiur" + if [ "$EVENT" = workflow_dispatch ]; then + # Reset the chosen workload(s) at the given commit; no PR scan. + sha="${INPUT_SHA:-$HEAD_SHA}" + [ "$INPUT_WORKLOAD" = all ] && workloads="$valid" || workloads="$INPUT_WORKLOAD" + else + # Reset the workloads the `ack` job recorded as `bencher-thresholds-reset:*` + # labels (already parsed/validated at comment time), anchored at the + # merge commit. Read straight from the event payload — no re-parsing. + # A `bencher-thresholds-reset:all` label expands to every workload. + sha="$MERGE_SHA" + workloads=$(jq -r '.pull_request.labels[].name + | select(startswith("bencher-thresholds-reset:")) | ltrimstr("bencher-thresholds-reset:")' \ + "$GITHUB_EVENT_PATH") + echo "$workloads" | grep -qx all && workloads="$valid" + fi + + done_list="" + for w in $workloads; do + case " $valid " in *" $w "*) ;; *) echo "skip unknown workload: $w"; continue ;; esac + tag="bencher-thresholds-reset-$w" + echo "Anchoring $tag -> $sha" + # Update the tag if it exists, else create it. (Checking first avoids + # the spurious 422 a PATCH-then-POST logs on first creation.) + if gh api --silent "repos/$REPO/git/refs/tags/$tag" 2>/dev/null; then + gh api --silent -X PATCH "repos/$REPO/git/refs/tags/$tag" -f sha="$sha" -F force=true + else + gh api --silent -X POST "repos/$REPO/git/refs" -f ref="refs/tags/$tag" -f sha="$sha" + fi + done_list="$done_list $w" + done + [ -z "$done_list" ] && { echo "Nothing to reset."; exit 0; } + + # Confirm on the PR for the merge-triggered path. + [ "$EVENT" = pull_request_target ] && gh pr comment "$PR" --repo "$REPO" \ + --body "♻️ Baseline reset to \`${sha:0:7}\` for:$done_list" + true + + # Immediate acknowledgment: when a maintainer comments the reset command on a + # still-open PR, reply that the named baseline(s) will be reset on merge — or + # warn if the workload is unknown/missing. The `user.type != 'Bot'` guard keeps + # this reply (which may echo the command) from re-triggering the job. + ack: + if: github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.issue.state == 'open' && github.event.comment.user.type != 'Bot' && contains(github.event.comment.body, '!bencher-thresholds-reset') && (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR') + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR: ${{ github.event.issue.number }} + BODY: ${{ github.event.comment.body }} + steps: + - run: | + # Accepted command tokens — applied verbatim as labels (incl. `all`, + # which the merge job expands into every workload). + accepted="ix-compile aiur all" + # Parse the workload token(s) after the command, lowercased. + workloads=$(printf '%s' "$BODY" \ + | grep -oiE '!bencher-thresholds-reset[[:space:]]+[a-z0-9 -]+' \ + | sed -E 's/^!bencher-thresholds-reset[[:space:]]+//I' \ + | tr '[:upper:] ' '[:lower:]\n' | sort -u) + ok="" + for w in $workloads; do + case " $accepted " in *" $w "*) ok="$ok $w" ;; *) ;; esac + done + if [ -n "$ok" ]; then + # Record each workload as a label — the flag the merge-time `reset` + # job reads. Create the label on first use (idempotent), then apply. + for w in $ok; do + label="bencher-thresholds-reset:$w" + gh label create "$label" --repo "$REPO" --color BFD4F2 \ + --description "bencher baseline reset queued on merge" 2>/dev/null || true + gh pr edit "$PR" --repo "$REPO" --add-label "$label" + done + gh pr comment "$PR" --repo "$REPO" \ + --body "♻️ Baseline reset queued for:$ok — will anchor to the merge commit when this PR merges." + else + gh pr comment "$PR" --repo "$REPO" \ + --body "⚠️ Reset command matched no known workload (expected \`ix-compile\`, \`aiur\`, or \`all\`). Nothing will reset on merge." + fi