diff --git a/.github/workflows/compute-projects-to-test/action.yml b/.github/workflows/compute-projects-to-test/action.yml new file mode 100644 index 0000000000000..37df06c8c301c --- /dev/null +++ b/.github/workflows/compute-projects-to-test/action.yml @@ -0,0 +1,21 @@ +name: 'Compute Projects To Test' +inputs: + projects: + required: false + type: 'string' + +outputs: + check-targets: + description: "A space delimited list of check-targets to pass to ninja." + value: ${{ steps.compute-projects.outputs.check-targets }} + + projects: + description: "A semi-colon delimited list of projects to pass to -DLLVM_ENABLE_PROJECTS." + value: ${{ steps.compute-projects.outputs.projects }} + +runs: + using: "composite" + steps: + - id: compute-projects + run: .github/workflows/compute-projects-to-test/compute-projects-to-test.sh ${{ inputs.projects }} + shell: bash diff --git a/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh new file mode 100755 index 0000000000000..4cfbda0c82034 --- /dev/null +++ b/.github/workflows/compute-projects-to-test/compute-projects-to-test.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +#===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===----------------------------------------------------------------------===## + +# +# This file generates a Buildkite pipeline that triggers the various CI jobs for +# the LLVM project during pre-commit CI. +# +# See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format. +# +# As this outputs a yaml file, it's possible to log messages to stderr or +# prefix with "#". + + +set -eu +set -o pipefail + +# Environment variables script works with: + +# Set by GitHub +: ${GITHUB_OUTPUT:=} +: ${RUNNER_OS:=} + +# Allow users to specify which projects to build. +all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" +if [ "$#" -ne 0 ]; then + wanted_projects="${@}" +else + wanted_projects="${all_projects}" +fi + +# List of files affected by this commit +: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1...HEAD)} + +echo "Files modified:" >&2 +echo "$MODIFIED_FILES" >&2 +modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u) +echo "Directories modified:" >&2 +echo "$modified_dirs" >&2 +echo "wanted_projects: $wanted_projects" + +function remove-unwanted-projects() { + projects=${@} + for project in ${projects}; do + if echo "$wanted_projects" | tr ' ' '\n' | grep -q -E "^${project}$"; then + echo "${project}" + fi + done +} + +function compute-projects-to-test() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + lld) + for p in bolt cross-project-tests; do + echo $p + done + ;; + llvm) + for p in bolt clang clang-tools-extra flang lld lldb mlir polly; do + echo $p + done + ;; + clang) + for p in clang-tools-extra compiler-rt flang libc lldb openmp cross-project-tests; do + echo $p + done + ;; + clang-tools-extra) + echo libc + ;; + mlir) + echo flang + ;; + *) + # Nothing to do + ;; + esac + done +} + +function add-dependencies() { + projects=${@} + for project in ${projects}; do + echo "${project}" + case ${project} in + bolt) + for p in lld llvm; do + echo $p + done + ;; + cross-project-tests) + for p in lld clang; do + echo $p + done + ;; + clang-tools-extra) + for p in llvm clang; do + echo $p + done + ;; + compiler-rt|libc|openmp) + echo clang lld + ;; + flang|lldb) + for p in llvm clang; do + echo $p + done + ;; + lld|mlir|polly) + echo llvm + ;; + *) + # Nothing to do + ;; + esac + done +} + +function exclude-linux() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + lldb) ;; # tests failing + openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410 + *) + echo "${project}" + ;; + esac + done +} + +function exclude-windows() { + projects=${@} + for project in ${projects}; do + case ${project} in + cross-project-tests) ;; # tests failing + compiler-rt) ;; # tests taking too long + openmp) ;; # TODO: having trouble with the Perl installation + libc) ;; # no Windows support + lldb) ;; # tests failing + bolt) ;; # tests are not supported yet + *) + echo "${project}" + ;; + esac + done +} + +# Prints only projects that are both present in $modified_dirs and the passed +# list. +function keep-modified-projects() { + projects=${@} + for project in ${projects}; do + if echo "$modified_dirs" | grep -q -E "^${project}$"; then + echo "${project}" + fi + done +} + +function check-targets() { + projects=${@} + for project in ${projects}; do + case ${project} in + clang-tools-extra) + echo "check-clang-tools" + ;; + compiler-rt) + echo "check-all" + ;; + cross-project-tests) + echo "check-cross-project" + ;; + lldb) + echo "check-all" # TODO: check-lldb may not include all the LLDB tests? + ;; + pstl) + echo "check-all" + ;; + libclc) + echo "check-all" + ;; + *) + echo "check-${project}" + ;; + esac + done +} + +# Generic pipeline for projects that have not defined custom steps. +# +# Individual projects should instead define the pre-commit CI tests that suits their +# needs while letting them run on the infrastructure provided by LLVM. + +# Figure out which projects need to be built on each platform +modified_projects="$(keep-modified-projects ${all_projects})" +echo "modified_projects: $modified_projects" + +if [ "${RUNNER_OS}" = "Linux" ]; then + projects_to_test=$(exclude-linux $(compute-projects-to-test ${modified_projects})) +elif [ "${RUNNER_OS}" = "Windows" ]; then + projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects})) +else + echo "Unknown runner OS: $RUNNER_OS" + exit 1 +fi +check_targets=$(check-targets $(remove-unwanted-projects ${projects_to_test}) | sort | uniq) +projects=$(remove-unwanted-projects $(add-dependencies ${projects_to_test}) | sort | uniq) + +echo "check-targets=$(echo ${check_targets} | tr ' ' ' ')" >> $GITHUB_OUTPUT +echo "projects=$(echo ${projects} | tr ' ' ';')" >> $GITHUB_OUTPUT + +cat $GITHUB_OUTPUT diff --git a/.github/workflows/continue-timeout-job.yml b/.github/workflows/continue-timeout-job.yml new file mode 100644 index 0000000000000..7f25a0a2adebd --- /dev/null +++ b/.github/workflows/continue-timeout-job.yml @@ -0,0 +1,70 @@ +name: Continue Timeout Job + +on: + workflow_run: + workflows: + - "Windows Precommit Tests" + types: + - completed + +permissions: + contents: read + +jobs: + restart: + name: "Restart Job" + permissions: + actions: write + runs-on: ubuntu-22.04 + if: github.event.workflow_run.conclusion == 'failure' + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/workflows/unprivileged-download-artifact + sparse-checkout-cone-mode: false + + - uses: ./.github/workflows/unprivileged-download-artifact + id: download-artifact + with: + run-id: ${{ github.event.workflow_run.id }} + artifact-name: timeout + + - shell: bash + if: steps.download-artifact.outputs.filename != '' + run: | + unzip ${{ steps.download-artifact.outputs.filename }} + + - name: "Restart Job" + if: steps.download-artifact.outputs.filename != '' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + with: + script: | + var fs = require('fs'); + const data = fs.readFileSync('./timeout'); + console.log(data); + const json = JSON.parse(data); + console.log(json); + if (!json || !json.job_id) { + console.log("Could not parse timeout artifact"); + return; + } + const job_id = json.job_id + + // Delete the timeout artifact to prepare for the next run + await github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: ${{ steps.download-artifact.outputs.artifact-id }} + }) + + // Restart the job + // This function does not exist even though it is in the document + //github.rest.actions.reRunJobForWorkflow({ + await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', { + owner: context.repo.owner, + repo: context.repo.repo, + job_id: job_id + }) + console.log("Restarted job: " + job_id); + return "timeout" diff --git a/.github/workflows/get-job-id/action.yml b/.github/workflows/get-job-id/action.yml new file mode 100644 index 0000000000000..65495efd86820 --- /dev/null +++ b/.github/workflows/get-job-id/action.yml @@ -0,0 +1,30 @@ +name: Get Job ID +inputs: + job-name: + required: false + type: 'string' + +outputs: + job-id: + description: "A space delimited list of check-targets to pass to ninja." + value: ${{ steps.job-id.outputs.result }} + +runs: + using: "composite" + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + id: job-id + with: + script: | + const job_data = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + }); + + for (job of job_data.data.jobs) { + console.log(job) + if (job.name == "${{ inputs.job-name }}") { + return job.id + } + } diff --git a/.github/workflows/pr-sccache-restore/action.yml b/.github/workflows/pr-sccache-restore/action.yml new file mode 100644 index 0000000000000..6c62d0e0de9f9 --- /dev/null +++ b/.github/workflows/pr-sccache-restore/action.yml @@ -0,0 +1,18 @@ +name: PR sccache restore + +runs: + using: "composite" + steps: + - uses: ./.github/workflows/unprivileged-download-artifact + id: download-artifact + with: + run-id: ${{ github.run_id }} + artifact-name: sccache-pr${{ github.event.pull_request.number }} + + - shell: bash + if: steps.download-artifact.outputs.filename != '' + run: | + # Is this the best way to clear the cache? + rm -Rf .sccache/ + unzip ${{ steps.download-artifact.outputs.filename }} + rm ${{ steps.download-artifact.outputs.filename }} diff --git a/.github/workflows/pr-sccache-save/action.yml b/.github/workflows/pr-sccache-save/action.yml new file mode 100644 index 0000000000000..def50fbc14d53 --- /dev/null +++ b/.github/workflows/pr-sccache-save/action.yml @@ -0,0 +1,40 @@ +name: PR sccache save + +runs: + using: "composite" + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + with: + script: | + const data = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + name: 'sccache-pr' + context.issue.number + }) + + console.log(data.data.artifacts) + if (data.data.artifacts.length == 0) { + return ''; + } + console.log(data.data.artifacts[0]) + const artifact_id = data.data.artifacts[0].id + + // Delete the exisiting artifact so we can upload a new one with the same name. + github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact_id + }) + + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + with: + name: 'sccache-pr${{ github.event.number }}' + path: .sccache + retention-days: 7 + + - shell: bash + run: | + sccache --show-stats + diff --git a/.github/workflows/precommit-windows.yml b/.github/workflows/precommit-windows.yml new file mode 100644 index 0000000000000..2ccf1fbee2270 --- /dev/null +++ b/.github/workflows/precommit-windows.yml @@ -0,0 +1,153 @@ +name: "Windows Precommit Tests" + +permissions: + contents: read + actions: write + +on: + pull_request: + types: + - opened + - synchronize + - reopened + # When a PR is closed, we still start this workflow, but then skip + # all the jobs, which makes it effectively a no-op. The reason to + # do this is that it allows us to take advantage of concurrency groups + # to cancel in progress CI jobs whenever the PR is closed. + - closed + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: True + +jobs: + compute-projects: + name: "Compute Projects to Test" + if: github.event.action != 'closed' + runs-on: ubuntu-22.04 + outputs: + projects: ${{ steps.vars.outputs.projects }} + check-targets: ${{ steps.vars.outputs.check-targets }} + test-build: ${{ steps.vars.outputs.check-targets != '' }} + steps: + - name: Fetch LLVM sources + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Compute projects to test + id: vars + uses: ./.github/workflows/compute-projects-to-test + + build-windows: + # If this job name is chagned, then we need to update the job-name + # paramater for the write-timeout-file step below. + name: "Build" + runs-on: windows-2022 + permissions: + actions: write #pr-sccache-save may delete artifacts. + outputs: + build-timeout: ${{ steps.build.outputs.timeout }} + build-failed: ${{ steps.build.outputs.failed }} + needs: + - compute-projects + if: ${{ needs.compute-projects.outputs.test-build == 'true' }} + steps: + - name: Download Artifact + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + pattern: timeout-build + merge-multiple: true + + - name: Unpack Artifact + id: timeout-artifact + shell: bash + run: | + if [ -e llvm-project.tar.zst ]; then + tar --zstd -xf llvm-project.tar.zst + rm llvm-project.tar.zst + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + - name: Setup Windows + uses: llvm/actions/setup-windows@main + with: + arch: amd64 + + - name: Fetch LLVM sources + if: ${{ steps.timeout-artifact.outputs.exists != 'true' }} + uses: actions/checkout@v4 + + - name: Setup sccache + uses: hendrikmuhs/ccache-action@v1 + with: + max-size: 2G + variant: sccache + key: precommit-windows + + - name: Restore sccache from previous PR run + uses: ./.github/workflows/pr-sccache-restore + + - name: Configure + if: ${{ steps.timeout-artifact.outputs.exists != 'true' }} + shell: bash + run: | + cmake -B build -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS="${{ needs.compute-projects.outputs.projects }}" \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_LIT_ARGS="-v --no-progress-bar" \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ + -S llvm + + - name: Write Timeout File + uses: ./.github/workflows/write-timeout-file + with: + job-name: "Build" + + - name: Build + shell: bash + id: build + timeout-minutes: 330 + run: | + touch timeout + ninja -C build -k 0 ${{ needs.compute-projects.outputs.check-targets }} && pass=1 + rm timeout + [ $pass ] || false + + - name: Upload Timeout Message + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + id: timeout + if: always() + with: + name: timeout + path: timeout + retention-days: 2 + + - name: Save sccache for next PR run + if: always() + uses: ./.github/workflows/pr-sccache-save + + - name: Package Build Directory + shell: bash + if: always() && steps.timeout.outputs.artifact-id != '' + run: | + # Remove the timeout file, so the next build isn't categorized + # as a timeout. + rm -f timeout + # Dereference symlinks so that this works on Windows. + tar -h -c . | zstd -T0 -c > ../llvm-project.tar.zst + mv ../llvm-project.tar.zst . + + - name: Upload Build + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + if: always() && steps.timeout.outputs.artifact-id != '' + with: + name: timeout-build + path: llvm-project.tar.zst + retention-days: 2 diff --git a/.github/workflows/unprivileged-download-artifact/action.yml b/.github/workflows/unprivileged-download-artifact/action.yml new file mode 100644 index 0000000000000..5e63f5ce8e3fe --- /dev/null +++ b/.github/workflows/unprivileged-download-artifact/action.yml @@ -0,0 +1,67 @@ +name: Unprivileged Download Artifact +description: Download artifacts from another workflow run without using an access token. +inputs: + run-id: + description: The run-id for the workflow run that you want to download the artifact from. + required: true + artifact-name: + desciption: The name of the artifact to download. + required: true + +outputs: + filename: + description: "The filename of the downloaded artifact or the empty string if the artifact was not found." + value: ${{ steps.download-artifact.outputs.filename }} + artifact-id: + description: "The id of the artifact being downloaded." + value: ${{ steps.artifact-url.outputs.id }} + + +runs: + using: "composite" + steps: + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 + id: artifact-url + with: + script: | + const response = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ inputs.run-id }}, + name: "${{ inputs.artifact-name }}" + }) + + console.log(response) + + for (artifact of response.data.artifacts) { + console.log(artifact); + } + + if (response.data.artifacts.length == 0) { + console.log("Could not find artifact ${{ inputs.artifact-name }} for workflow run ${{ inputs.run-id }}") + return; + } + + const url_response = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: response.data.artifacts[0].id, + archive_format: "zip" + }) + + core.setOutput("url", url_response.url); + core.setOutput("id", response.data.artifacts[0].id); + + - shell: bash + run: | + echo "${{ steps.artifact-url.outputs.url }}" + echo "${{ steps.artifact-url.outputs.result }}" + echo "${{ steps.artifact-url.outputs.result.url }}" + echo "${{ steps.artifact-url.outputs.result.id }}" + + - shell: bash + if: steps.artifact-url.outputs.url != '' + id: download-artifact + run: | + curl -L -o ${{ inputs.artifact-name }}.zip "${{ steps.artifact-url.outputs.url }}" + echo "filename=${{ inputs.artifact-name }}.zip" >> $GITHUB_OUTPUT diff --git a/.github/workflows/write-timeout-file/action.yml b/.github/workflows/write-timeout-file/action.yml new file mode 100644 index 0000000000000..a483185ab623f --- /dev/null +++ b/.github/workflows/write-timeout-file/action.yml @@ -0,0 +1,17 @@ +name: Write Timeout File +inputs: + job-name: + required: false + type: 'string' + +runs: + using: "composite" + steps: + - uses: ./.github/workflows/get-job-id + id: job-id + with: + job-name: ${{ inputs.job-name }} + + - shell: bash + run: | + echo '{"job_id": "${{ steps.job-id.outputs.job-id }}"}' > timeout