ci-results.yaml

# publishes test results from the CI workflow (not when run on schedule)
# this publishes test results of PRs from horovod repository and fork repositories
# buildkite tests are only run here for fork repositories
name: CI (Results)

on:
  workflow_run:
    workflows: ["CI"]
    types:
      - completed

permissions: {}

jobs:
  ci-workflow:
    name: "Check CI workflow outcome"
    runs-on: ubuntu-latest
    # only run if CI workflow has not been skipped or cancelled
    # only run if CI workflow did not run on schedule
    if: >
      github.event.workflow_run.conclusion != 'skipped' &&
      github.event.workflow_run.conclusion != 'cancelled' &&
      github.event.workflow_run.event != 'schedule'
    permissions:
      # steps "Fetch workflow conclusion" and "Fetch PR meta"
      actions: read
      # step "Fetch PR meta"
      contents: read
    outputs:
      build-and-test: ${{ steps.workflow-conclusion.outputs.build-and-test }}
      pr-json: ${{ steps.pr.outputs.json }}

    steps:
    - name: Fetch workflow conclusion
      # fetch conclusion of steps building and testing CPU and building GPU
      # ignores steps building heads and mins, building and testing macOS, building and testing GPU via Buildkite
      id: workflow-conclusion
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        conclusion=$(gh api --paginate "${{ github.event.workflow_run.jobs_url }}" -q '.jobs[] | select(.name | startswith("Build and Test (")) | .conclusion' | sort | uniq | paste -sd "," -)
        echo "build-and-test conclusion: ${conclusion}"
        echo "build-and-test=${conclusion}" >> $GITHUB_OUTPUT
      shell: bash

    - name: Fetch PR meta
      id: pr
      if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.fork
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        artifacts_url=${{ github.event.workflow_run.artifacts_url }}
        gh api --paginate "$artifacts_url" -q '.artifacts[] | select(.name == "PR Meta") .archive_download_url' | while read url
        do
          gh api "$url" > "pr.zip"
          unzip -o "pr.zip"
          echo "json=$(cat pr.json)" >> $GITHUB_OUTPUT
          cat pr.json
          echo
        done

        if [[ ! -e "pr.json" ]]
        then
          echo "::error title=Artifact 'PR Meta' missing::Expected artifact 'PR Meta' does not exist for pull_request event."
          exit 1
        fi

  buildkite-trigger:
    name: "Build and Test GPU (trigger Builtkite)"
    needs: [ci-workflow]
    runs-on: ubuntu-latest
    # only run if CI workflow's build-and-test job succeeded and CI workflow ran on a fork
    if: >
      needs.ci-workflow.outputs.build-and-test == 'success' &&
      github.event.workflow_run.head_repository.fork
    outputs:
      url: ${{ steps.build.outputs.url }}
    permissions:
      # step "Create check status"
      statuses: write

    steps:
      - name: Create check status
        continue-on-error: true
        run: |
          curl --request POST \
          --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_commit.id }} \
          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
          --header 'content-type: application/json' \
          --data "{
            \"state\": \"pending\",
            \"context\": \"${GITHUB_WORKFLOW} / Build and Test GPU (on Builtkite)\",
            \"target_url\": \"https://github.com/horovod/horovod/actions/runs/${GITHUB_RUN_ID}\"
          }"

      - name: Trigger Buildkite Pipeline
        id: build
        uses: EnricoMi/trigger-pipeline-action@master
        env:
          PIPELINE: "horovod/horovod"
          COMMIT: "${{ fromJSON( needs.ci-workflow.outputs.pr-json ).merge_sha }}"
          BRANCH: "${{ github.event.workflow_run.head_repository.owner.login }}:${{ github.event.workflow_run.head_branch }} (GPU NON HEADS)"
          MESSAGE: "${{ github.event.workflow_run.head_commit.message }}"
          BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
          BUILD_ENV_VARS: "{\"PIPELINE_MODE\": \"GPU NON HEADS\"}"

  buildkite:
    name: "Build and Test GPU (download Builtkite)"
    needs: [buildkite-trigger]
    runs-on: ubuntu-latest
    permissions:
      # step "Update check status"
      statuses: write

    steps:
      - name: Download Buildkite Artifacts
        id: download
        uses: EnricoMi/download-buildkite-artifact-action@v1
        with:
          github_token: ${{ github.token }}
          buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
          buildkite_build_url: ${{ needs.buildkite-trigger.outputs.url }}
          ignore_build_states: blocked,canceled,skipped,not_run
          ignore_job_states: timed_out
          output_path: artifacts/Unit Test Results - GPU NON HEADS on Builtkite

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: Unit Test Results - GPU NON HEADS on Builtkite
          path: artifacts/Unit Test Results - GPU NON HEADS on Builtkite/**/*.xml

      - name: Check Buildkite job state
        if: >
          always() &&
          steps.download.conclusion == 'success' &&
          steps.download.outputs.build-state != 'passed'
        run: |
          echo "::warning::Buildkite pipeline did not pass: ${{ needs.buildkite-trigger.outputs.url }}"
          exit 1

      - name: Update check status
        # job status can be success, failure, or cancelled but status state only allows for error, failure, pending, or success
        if: always() && job.status != 'cancelled'
        continue-on-error: true
        run: |
          curl --request POST \
          --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_commit.id }} \
          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
          --header 'content-type: application/json' \
          --data "{
            \"state\": \"${{ job.status }}\",
            \"context\": \"${GITHUB_WORKFLOW} / Build and Test GPU (on Builtkite)\",
            \"target_url\": \"https://github.com/horovod/horovod/actions/runs/${GITHUB_RUN_ID}\"
          }"

  buildkite-heads-trigger:
    name: "Build and Test GPU heads (trigger Builtkite)"
    needs: [ci-workflow]
    runs-on: ubuntu-latest
    # only run if CI workflow's build-and-test job succeeded and CI workflow ran on a fork
    if: >
      needs.ci-workflow.outputs.build-and-test == 'success' &&
      github.event.workflow_run.head_repository.fork
    outputs:
      url: ${{ steps.build.outputs.url }}
    permissions:
      # step "Create check status"
      statuses: write

    steps:
      - name: Create check status
        continue-on-error: true
        run: |
          curl --request POST \
          --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_commit.id }} \
          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
          --header 'content-type: application/json' \
          --data "{
            \"state\": \"pending\",
            \"context\": \"${GITHUB_WORKFLOW} / Build and Test GPU heads (on Builtkite)\",
            \"target_url\": \"https://github.com/horovod/horovod/actions/runs/${GITHUB_RUN_ID}\"
          }"

      - name: Trigger Buildkite Pipeline
        id: build
        uses: EnricoMi/trigger-pipeline-action@master
        env:
          PIPELINE: "horovod/horovod"
          COMMIT: "${{ fromJSON( needs.ci-workflow.outputs.pr-json ).merge_sha }}"
          BRANCH: "${{ github.event.workflow_run.head_repository.owner.login }}:${{ github.event.workflow_run.head_branch }} (GPU HEADS)"
          MESSAGE: "${{ github.event.workflow_run.head_commit.message }}"
          BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
          BUILD_ENV_VARS: "{\"PIPELINE_MODE\": \"GPU HEADS\"}"

  buildkite-heads:
    name: "Build and Test GPU heads (download Builtkite)"
    needs: [buildkite-heads-trigger]
    runs-on: ubuntu-latest
    permissions:
      # step "Update check status"
      statuses: write

    steps:
      - name: Download Buildkite Artifacts
        id: download
        uses: EnricoMi/download-buildkite-artifact-action@v1
        with:
          github_token: ${{ github.token }}
          buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
          buildkite_build_url: ${{ needs.buildkite-heads-trigger.outputs.url }}
          ignore_build_states: blocked,canceled,skipped,not_run
          ignore_job_states: timed_out
          output_path: artifacts/Unit Test Results - GPU HEADS on Builtkite

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: Unit Test Results - GPU HEADS on Builtkite
          path: artifacts/Unit Test Results - GPU HEADS on Builtkite/**/*.xml

      - name: Check Buildkite job state
        if: >
          always() &&
          steps.download.conclusion == 'success' &&
          steps.download.outputs.build-state != 'passed'
        run: |
          echo "::warning::Buildkite pipeline did not pass: ${{ needs.buildkite-heads-trigger.outputs.url }}"
          exit 1

      - name: Update check status
        # job status can be success, failure, or cancelled but status state only allows for error, failure, pending, or success
        if: always() && job.status != 'cancelled'
        continue-on-error: true
        run: |
          curl --request POST \
          --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_commit.id }} \
          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
          --header 'content-type: application/json' \
          --data "{
            \"state\": \"${{ job.status }}\",
            \"context\": \"${GITHUB_WORKFLOW} / Build and Test GPU heads (on Builtkite)\",
            \"target_url\": \"https://github.com/horovod/horovod/actions/runs/${GITHUB_RUN_ID}\"
          }"

  publish-test-results:
    name: "Publish Unit Tests Results"
    needs: [ci-workflow, buildkite, buildkite-heads]
    runs-on: ubuntu-latest
    # only publish results when ci-workflow job has not been skipped, meaning:
    # - CI workflow has not been skipped or cancelled
    # - CI workflow did not run on schedule
    # and CI workflow's build-and-test jobs have not all been skipped
    if: >
      always() &&
      needs.ci-workflow.result != 'skipped' &&
      needs.ci-workflow.outputs.build-and-test != 'skipped'
    permissions:
      # step "Download and Extract Artifacts"
      actions: read
      contents: read

      # steps "Publish Unit Test Results" and "Publish Unit Test Results (with flaky tests)"
      checks: write
      pull-requests: write
    steps:
      - name: Download and Extract Artifacts
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          mkdir -p artifacts && cd artifacts

          artifacts_url=${{ github.event.workflow_run.artifacts_url }}

          gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
          do
            IFS=$'\t' read name url <<< "$artifact"
            gh api $url > "$name.zip"
            unzip -d "$name" "$name.zip"
          done

      - name: Download Buildkite Artifacts
        uses: actions/download-artifact@v3
        with:
          path: artifacts

      - name: Identify last run of each test
        continue-on-error: true
        run: |
          declare -A last_runs
          ls -d artifacts/Unit\ Test\ Results\ */* | sort > runs.txt
          while read run
          do
            test=${run/%[_-]run[_-][0123456789]/}
            last_runs[$test]=$run
          done < runs.txt

          echo "LAST_RUNS<<EOF" >> $GITHUB_ENV
          for test in "${!last_runs[@]}"
          do
            echo "${last_runs[$test]}" >&2
            echo "${last_runs[$test]}/**/*.xml" >> $GITHUB_ENV
          done
          echo "EOF" >> $GITHUB_ENV
        shell: bash

      - name: Publish Unit Test Results
        uses: EnricoMi/publish-unit-test-result-action@v2
        if: always()
        with:
          check_name: Unit Test Results
          event_file: artifacts/Event File/event.json
          event_name: ${{ github.event.workflow_run.event }}
          commit: ${{ github.event.workflow_run.head_sha }}
          junit_files: "${{ env.LAST_RUNS }}"
          log_level: DEBUG

      - name: Publish Unit Test Results (with flaky tests)
        uses: EnricoMi/publish-unit-test-result-action@v2
        if: always()
        with:
          check_name: Unit Test Results (with flaky tests)
          event_file: artifacts/Event File/event.json
          event_name: ${{ github.event.workflow_run.event }}
          commit: ${{ github.event.workflow_run.head_sha }}
          junit_files: "artifacts/Unit Test Results */**/*.xml"
          log_level: DEBUG
          fail_on: errors