diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16b0512a88a46..90269f56c52a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,8 +36,21 @@ concurrency: group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}" cancel-in-progress: true jobs: + calculate_matrix: + name: Calculate job matrix + runs-on: ubuntu-latest + outputs: + jobs: "${{ steps.jobs.outputs.jobs }}" + steps: + - name: Checkout the source code + uses: actions/checkout@v4 + - name: Calculate the CI job matrix + run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + id: jobs pr: name: "PR - ${{ matrix.name }}" + needs: + - calculate_matrix env: PR_CI_JOB: 1 CI_JOB_NAME: "${{ matrix.name }}" @@ -51,20 +64,7 @@ jobs: continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}" strategy: matrix: - include: - - name: mingw-check - os: ubuntu-20.04-4core-16gb - env: {} - - name: mingw-check-tidy - os: ubuntu-20.04-4core-16gb - env: {} - - name: x86_64-gnu-llvm-17 - env: - ENABLE_GCC_CODEGEN: "1" - os: ubuntu-20.04-16core-64gb - - name: x86_64-gnu-tools - os: ubuntu-20.04-16core-64gb - env: {} + include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}" defaults: run: shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 9323bb093ad96..de71b9f874f8d 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -341,9 +341,22 @@ concurrency: cancel-in-progress: true jobs: + # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. + calculate_matrix: + name: Calculate job matrix + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.jobs.outputs.jobs }} + steps: + - name: Checkout the source code + uses: actions/checkout@v4 + - name: Calculate the CI job matrix + run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + id: jobs pr: <<: *base-ci-job name: PR - ${{ matrix.name }} + needs: [ calculate_matrix ] env: <<: [*shared-ci-variables, *public-variables] PR_CI_JOB: 1 @@ -351,20 +364,8 @@ jobs: continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }} strategy: matrix: - include: - - name: mingw-check - <<: *job-linux-4c - - - name: mingw-check-tidy - <<: *job-linux-4c - - - name: x86_64-gnu-llvm-17 - env: - ENABLE_GCC_CODEGEN: "1" - <<: *job-linux-16c - - - name: x86_64-gnu-tools - <<: *job-linux-16c + # Check the `calculate_matrix` job to see how is the matrix defined. + include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} auto: <<: *base-ci-job diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml new file mode 100644 index 0000000000000..7e89eef2670e4 --- /dev/null +++ b/src/ci/github-actions/jobs.yml @@ -0,0 +1,50 @@ +# This file contains definitions of CI job parameters that are loaded +# dynamically in CI from ci.yml. +# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you +# modify this file. +shared_defs: + - &base-job + env: { } + + - &job-linux-4c + os: ubuntu-20.04-4core-16gb + <<: *base-job + + - &job-linux-8c + os: ubuntu-20.04-8core-32gb + <<: *base-job + + - &job-linux-16c + os: ubuntu-20.04-16core-64gb + <<: *base-job + + - &job-macos-xl + os: macos-13 # We use the standard runner for now + <<: *base-job + + - &job-macos-m1 + os: macos-14 + <<: *base-job + + - &job-windows-8c + os: windows-2019-8core-32gb + <<: *base-job + + - &job-windows-16c + os: windows-2019-16core-64gb + <<: *base-job + + - &job-aarch64-linux + os: [ self-hosted, ARM64, linux ] + +pr: + - name: mingw-check + <<: *job-linux-4c + - name: mingw-check-tidy + <<: *job-linux-4c + - name: x86_64-gnu-llvm-17 + env: + ENABLE_GCC_CODEGEN: "1" + <<: *job-linux-16c + - name: x86_64-gnu-tools + <<: *job-linux-16c diff --git a/src/ci/scripts/calculate-job-matrix.py b/src/ci/scripts/calculate-job-matrix.py new file mode 100755 index 0000000000000..9b1e74c23c324 --- /dev/null +++ b/src/ci/scripts/calculate-job-matrix.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +""" +This script serves for generating a matrix of jobs that should +be executed on CI. + +It reads job definitions from `src/ci/github-actions/jobs.yml` +and filters them based on the event that happened on CI. + +Currently, it only supports PR builds. +""" + +import json +from pathlib import Path + +import yaml + +JOBS_YAML_PATH = Path(__file__).absolute().parent.parent / "github-actions" / "jobs.yml" + + +if __name__ == "__main__": + with open(JOBS_YAML_PATH) as f: + jobs = yaml.safe_load(f) + job_output = jobs["pr"] + print(f"jobs={json.dumps(job_output)}") diff --git a/src/tools/expand-yaml-anchors/src/main.rs b/src/tools/expand-yaml-anchors/src/main.rs index 3fc72ecbbc484..60651734b9e30 100644 --- a/src/tools/expand-yaml-anchors/src/main.rs +++ b/src/tools/expand-yaml-anchors/src/main.rs @@ -2,11 +2,11 @@ use std::error::Error; use std::path::{Path, PathBuf}; use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; -/// List of directories containing files to expand. The first tuple element is the source -/// directory, while the second tuple element is the destination directory. +/// List of files to expand. The first tuple element is the source +/// file, while the second tuple element is the destination file. #[rustfmt::skip] static TO_EXPAND: &[(&str, &str)] = &[ - ("src/ci/github-actions", ".github/workflows"), + ("src/ci/github-actions/ci.yml", ".github/workflows/ci.yml"), ]; /// Name of a special key that will be removed from all the maps in expanded configuration files. @@ -62,27 +62,20 @@ impl App { fn run(&self) -> Result<(), Box> { for (source, dest) in TO_EXPAND { let source = self.base.join(source); - let dest = self.base.join(dest); - for entry in std::fs::read_dir(&source)? { - let path = entry?.path(); - if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") { - continue; - } - - let dest_path = dest.join(path.file_name().unwrap()); - self.expand(&path, &dest_path).with_context(|| match self.mode { - Mode::Generate => format!( - "failed to expand {} into {}", - self.path(&path), - self.path(&dest_path) - ), - Mode::Check => format!( - "{} is not up to date; please run \ + let dest_path = self.base.join(dest); + + self.expand(&source, &dest_path).with_context(|| match self.mode { + Mode::Generate => format!( + "failed to expand {} into {}", + self.path(&source), + self.path(&dest_path) + ), + Mode::Check => format!( + "{} is not up to date; please run \ `x.py run src/tools/expand-yaml-anchors`.", - self.path(&dest_path) - ), - })?; - } + self.path(&dest_path) + ), + })?; } Ok(()) }