-
Notifications
You must be signed in to change notification settings - Fork 11.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Upstream libc++ buildbot restarter. (#93582)
I've been running a cronjob on my local machine to restart preempted libc++ CI runs. This is bad and brittle. This upstreams a much better version of the restarter. It works by matching on check run annotations looking for mention of the machine being shutdown. If there are both preempted jobs and failing jobs, we don't restart the workflow. Maybe we should change that?
- Loading branch information
Showing
1 changed file
with
109 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
name: Restart Preempted Libc++ Workflow | ||
|
||
# The libc++ builders run on preemptable VMs, which can be shutdown at any time. | ||
# This workflow identifies when a workflow run was canceled due to the VM being preempted, | ||
# and restarts the workflow run. | ||
|
||
# We identify a canceled workflow run by checking the annotations of the check runs in the check suite, | ||
# which should contain the message "The runner has received a shutdown signal." | ||
|
||
# Note: If a job is both preempted and also contains a non-preemption failure, we do not restart the workflow. | ||
|
||
on: | ||
workflow_run: | ||
workflows: | ||
- "Build and Test libc\+\+" | ||
types: | ||
- failure | ||
- canceled | ||
|
||
permissions: | ||
contents: read | ||
|
||
jobs: | ||
restart: | ||
if: github.repository_owner == 'llvm' | ||
name: "Restart Job" | ||
permissions: | ||
statuses: read | ||
checks: read | ||
actions: write | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: "Restart Job" | ||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 | ||
with: | ||
script: | | ||
const failure_regex = /Process completed with exit code 1./ | ||
const preemption_regex = /The runner has received a shutdown signal/ | ||
console.log('Listing check runs for suite') | ||
const check_suites = await github.rest.checks.listForSuite({ | ||
owner: context.repo.owner, | ||
repo: context.repo.repo, | ||
check_suite_id: context.payload.workflow_run.check_suite_id | ||
}) | ||
check_run_ids = []; | ||
for (check_run of check_suites.data.check_runs) { | ||
console.log('Checking check run: ' + check_run.id); | ||
console.log(check_run); | ||
if (check_run.status != 'completed') { | ||
console.log('Check run was not completed. Skipping.'); | ||
continue; | ||
} | ||
if (check_run.conclusion != 'failure' && check_run.conclusion != 'cancelled') { | ||
console.log('Check run had conclusion: ' + check_run.conclusion + '. Skipping.'); | ||
continue; | ||
} | ||
check_run_ids.push(check_run.id); | ||
} | ||
has_preempted_job = false; | ||
for (check_run_id of check_run_ids) { | ||
console.log('Listing annotations for check run: ' + check_run_id); | ||
annotations = await github.rest.checks.listAnnotations({ | ||
owner: context.repo.owner, | ||
repo: context.repo.repo, | ||
check_run_id: check_run_id | ||
}) | ||
console.log(annotations); | ||
for (annotation of annotations.data) { | ||
if (annotation.annotation_level != 'failure') { | ||
continue; | ||
} | ||
const preemption_match = annotation.message.match(preemption_regex); | ||
if (preemption_match != null) { | ||
console.log('Found preemption message: ' + annotation.message); | ||
has_preempted_job = true; | ||
} | ||
const failure_match = annotation.message.match(failure_regex); | ||
if (failure_match != null) { | ||
// We only want to restart the workflow if all of the failures were due to preemption. | ||
// We don't want to restart the workflow if there were other failures. | ||
console.log('Choosing not to rerun workflow because we found a non-preemption failure'); | ||
console.log('Failure message: ' + annotation.message); | ||
return; | ||
} | ||
} | ||
} | ||
if (!has_preempted_job) { | ||
console.log('No preempted jobs found. Not restarting workflow.'); | ||
return; | ||
} | ||
console.log("Restarted workflow: " + context.payload.workflow_run.id); | ||
await github.rest.actions.reRunWorkflowFailedJobs({ | ||
owner: context.repo.owner, | ||
repo: context.repo.repo, | ||
run_id: context.payload.workflow_run.id | ||
}) | ||