Skip to content

ci: cap NVHPC lane concurrency with max-parallel #1826

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sbryngelson wants to merge 4 commits into
base: master
Choose a base branch
from
+131 −119
Open

ci: cap NVHPC lane concurrency with max-parallel #1826

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e42d2f8
ci: cap NVHPC lane concurrency with max-parallel
sbryngelson Sep 5, 2026
4acb93c
ci: declare permissions in test.yml so the new nvhpc job is scoped
sbryngelson Sep 5, 2026
73edd77
ci: scope pull-requests: read to the file-changes job
sbryngelson Sep 5, 2026
0e5f9fd
ci: drop --test-all from the NVHPC cpu test step
sbryngelson Sep 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 131 additions & 119 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
cancel-in-progress: ${{ github.event_name != 'push' }}

# Least-privilege default: no job in this workflow writes to the repo.
permissions:
contents: read

jobs:
lint-gate:
name: Lint Gate
Expand Down Expand Up @@ -50,6 +54,13 @@ jobs:

file-changes:
name: Detect File Changes
# Job-level permissions replace the workflow default outright rather than
# merging with it, so contents must be restated here. paths-filter reads the
# PR's changed-file list via pulls.listFiles; this is the only job that needs
# it, so it is granted here instead of workflow-wide.
permissions:
contents: read
pull-requests: read
runs-on: 'ubuntu-latest'
outputs:
checkall: ${{ steps.changes.outputs.checkall }}
Expand All @@ -66,7 +77,7 @@ jobs:
list-files: shell

github:
name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, {3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel && 'intel' || 'GNU') }}
name: ${{ format('Github ({0}, {1}, {2}, {3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel && 'intel' || 'GNU') }}
needs: [lint-gate, file-changes]
if: >-
!cancelled() &&
Expand All @@ -80,8 +91,6 @@ jobs:
precision: ['']
debug: ['reldebug', 'no-debug']
intel: [true, false]
nvhpc: ['']
target: ['']
exclude:
- os: macos
intel: true
Expand All @@ -93,50 +102,113 @@ jobs:
debug: no-debug
intel: false

# NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp)
# Every release from 23.11 through 26.3 (current)
- { nvhpc: '23.11', target: cpu }
- { nvhpc: '23.11', target: gpu }
- { nvhpc: '24.1', target: cpu }
- { nvhpc: '24.1', target: gpu }
- { nvhpc: '24.3', target: cpu }
- { nvhpc: '24.3', target: gpu }
- { nvhpc: '24.5', target: cpu }
- { nvhpc: '24.5', target: gpu }
- { nvhpc: '24.7', target: cpu }
- { nvhpc: '24.7', target: gpu }
- { nvhpc: '24.9', target: cpu }
- { nvhpc: '24.9', target: gpu }
- { nvhpc: '24.11', target: cpu }
- { nvhpc: '24.11', target: gpu }
- { nvhpc: '25.1', target: cpu }
- { nvhpc: '25.1', target: gpu }
- { nvhpc: '25.3', target: cpu }
- { nvhpc: '25.3', target: gpu }
- { nvhpc: '25.5', target: cpu }
- { nvhpc: '25.5', target: gpu }
- { nvhpc: '25.7', target: cpu }
- { nvhpc: '25.7', target: gpu }
- { nvhpc: '25.9', target: cpu }
- { nvhpc: '25.9', target: gpu }
- { nvhpc: '25.11', target: cpu }
- { nvhpc: '25.11', target: gpu }
- { nvhpc: '26.1', target: cpu }
- { nvhpc: '26.1', target: gpu }
- { nvhpc: '26.3', target: cpu }
- { nvhpc: '26.3', target: gpu }
fail-fast: false
continue-on-error: true
runs-on: ${{ format('{0}-latest', matrix.os) }}

steps:
- name: Clone
uses: actions/checkout@v5

# ── Standard (non-NVHPC) setup ─────────────────────────────────────
- name: Setup MacOS
if: matrix.os == 'macos'
run: |
brew update
brew upgrade || true
brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack
echo "FC=gfortran-15" >> $GITHUB_ENV
echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV

- name: Setup Ubuntu
if: matrix.os == 'ubuntu' && matrix.intel == false
run: |
sudo apt update -y
sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \
libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \
libblas-dev liblapack-dev

- name: Setup Ubuntu (Intel)
if: matrix.os == 'ubuntu' && matrix.intel == true
run: |
wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
sudo apt-get update
sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel
# Export only new/changed env vars from setvars.sh.
# `printenv >> $GITHUB_ENV` dumps all vars including shell internals
# with special characters that corrupt GITHUB_ENV parsing.
printenv | sort > /tmp/env_before
source /opt/intel/oneapi/setvars.sh
printenv | sort > /tmp/env_after
diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV
echo "FC=ifx" >> $GITHUB_ENV
echo "CC=icx" >> $GITHUB_ENV
echo "CXX=icpx" >> $GITHUB_ENV
echo "MPIFC=mpiifx" >> $GITHUB_ENV
echo "MPICC=mpiicx" >> $GITHUB_ENV
echo "MPICXX=mpiicpx" >> $GITHUB_ENV

# ── Standard build + test ───────────────────────────────────────────
- name: Build
run: |
/bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL
env:
TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }}

- name: Test
run: |
# Coverage-based test selection ENFORCED on PRs: runs only the tests whose
# recorded coverage overlaps the PR's changed files (conservative ladder in
# coverage.py — non-.fpp changes and .fpp files no test covers fall back to
# run-all). Pushes to master run the full suite (SELECT empty) as a backstop,
# and the nvhpc job still runs --test-all as a pre-merge full check.
Comment on lines +163 to +167
SELECT=()
[ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--select-enforce --changed-files "$CHANGED_FILES")
/bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION
env:
TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }}
PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }}
CHANGED_FILES: ${{ needs.file-changes.outputs.changed_files }}

# Split out of the `github` job so max-parallel can throttle these lanes
# without also throttling the fast ubuntu/macos ones: max-parallel applies to
# a whole job matrix, and the two sets used to share one.
nvhpc:
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
name: NVHPC ${{ matrix.nvhpc }} (${{ matrix.target }})
needs: [lint-gate, file-changes]
if: >-
!cancelled() &&
needs.lint-gate.result == 'success' &&
needs.file-changes.result == 'success' &&
needs.file-changes.outputs.checkall == 'true'
strategy:
fail-fast: false
# Cap concurrent runners. These 30 lanes are advisory compile coverage
# (continue-on-error), but dispatching them all at once consumed the
# account's standard-runner concurrency and left other workflows queued
# behind them. A queued matrix entry holds no runner, so this buys slots
# back at the cost of wall-clock. It also staggers the nvcr.io pulls that
# the retry loop below exists to survive.
max-parallel: 6
matrix:
# NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp)
# Every release from 23.11 through 26.3 (current)
nvhpc: ['23.11', '24.1', '24.3', '24.5', '24.7', '24.9', '24.11',
'25.1', '25.3', '25.5', '25.7', '25.9', '25.11',
'26.1', '26.3']
target: [cpu, gpu]
continue-on-error: true
runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }}
runs-on: ubuntu-22.04
env:
# Image tag for NVHPC jobs; empty for non-NVHPC jobs.
NVHPC_IMAGE: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }}
NVHPC_IMAGE: nvcr.io/nvidia/nvhpc:${{ matrix.nvhpc }}-devel-cuda_multi-ubuntu22.04

steps:
# ── NVHPC: free disk before pulling the ~25-30 GB cuda_multi image ──
- name: Free disk space
if: matrix.nvhpc
run: |
echo "=== Disk before cleanup ==="
df -h /
Expand All @@ -160,7 +232,6 @@ jobs:
# exceeded") under load, and ~30 matrix jobs hit it at once. Pulls
# resume completed layers, so retries are cheap.
- name: Pull NVHPC container
if: matrix.nvhpc
run: |
for attempt in 1 2 3 4 5; do
docker pull "$NVHPC_IMAGE" && exit 0
Expand All @@ -171,7 +242,6 @@ jobs:
exit 1

- name: Start NVHPC container
if: matrix.nvhpc
run: |
docker run -d --name nvhpc \
--security-opt seccomp=unconfined \
Expand All @@ -187,11 +257,10 @@ jobs:
-e "FFLAGS=-tp=px -Kieee -noswitcherror" \
-e CFLAGS=-tp=px \
-e CXXFLAGS=-tp=px \
-e "MFC_NVHPC_TEST_FLAGS=--test-all --no-mpi" \
-e "MFC_NVHPC_TEST_FLAGS=--no-mpi" \
"$NVHPC_IMAGE" sleep infinity

- name: Setup NVHPC
if: matrix.nvhpc
run: |
docker exec nvhpc bash -c '
set -e
Expand Down Expand Up @@ -223,90 +292,33 @@ jobs:
cat /proc/cpuinfo | grep "model name" | head -1
'

# ── Standard (non-NVHPC) setup ─────────────────────────────────────
- name: Setup MacOS
if: matrix.os == 'macos' && !matrix.nvhpc
run: |
brew update
brew upgrade || true
brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack
echo "FC=gfortran-15" >> $GITHUB_ENV
echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV

- name: Setup Ubuntu
if: matrix.os == 'ubuntu' && matrix.intel == false && !matrix.nvhpc
run: |
sudo apt update -y
sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \
libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \
libblas-dev liblapack-dev

- name: Setup Ubuntu (Intel)
if: matrix.os == 'ubuntu' && matrix.intel == true
run: |
wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
sudo apt-get update
sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel
# Export only new/changed env vars from setvars.sh.
# `printenv >> $GITHUB_ENV` dumps all vars including shell internals
# with special characters that corrupt GITHUB_ENV parsing.
printenv | sort > /tmp/env_before
source /opt/intel/oneapi/setvars.sh
printenv | sort > /tmp/env_after
diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV
echo "FC=ifx" >> $GITHUB_ENV
echo "CC=icx" >> $GITHUB_ENV
echo "CXX=icpx" >> $GITHUB_ENV
echo "MPIFC=mpiifx" >> $GITHUB_ENV
echo "MPICC=mpiicx" >> $GITHUB_ENV
echo "MPICXX=mpiicpx" >> $GITHUB_ENV

# ── Standard build + test ───────────────────────────────────────────
- name: Build
if: '!matrix.nvhpc'
run: |
/bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL
env:
TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }}

- name: Test
if: '!matrix.nvhpc'
run: |
# Coverage-based test selection ENFORCED on PRs: runs only the tests whose
# recorded coverage overlaps the PR's changed files (conservative ladder in
# coverage.py — non-.fpp changes and .fpp files no test covers fall back to
# run-all). Pushes to master run the full suite (SELECT empty) as a backstop,
# and the NVHPC jobs below still run --test-all as a pre-merge full check.
SELECT=()
[ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--select-enforce --changed-files "$CHANGED_FILES")
/bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION
env:
TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }}
PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }}
CHANGED_FILES: ${{ needs.file-changes.outputs.changed_files }}

# ── NVHPC build + test (via docker exec into long-lived container) ──
# MFC_NVHPC_TEST_FLAGS carries --no-mpi: post_process segfaults under
# mpirun in these containers, and every failure the lanes have produced
# has been an MPI test (see PR #1822). The harness skips ppn>1 cases
# without MPI, so the rest of --test-all -- the compile coverage these
# lanes exist for -- is unaffected, as is MPI coverage elsewhere. Set in
# one place so the build and test steps cannot disagree: a no-MPI build
# tested with MPI would run ppn>1 cases against a binary that has none.
# without MPI, so compile coverage is unaffected, as is MPI coverage
# elsewhere. It stays in one place because a no-MPI build tested with
# MPI would run ppn>1 cases against a binary that has none.
#
# --test-all is deliberately NOT shared, and sits on the build only.
# It does two separate things (toolchain/mfc/test/test.py): at build
# time it adds post_process to the set of compiled binaries, and at
# test time it re-runs pre_process+simulation+post_process a second
# time for every case. The first is the compile coverage these lanes
# exist for; the second is roughly half the lane's ~120 min wall clock
# and duplicates runtime checking the GNU, Intel and self-hosted lanes
# already do. So: build with it, test without it. post_process is
# still compiled on all 15 releases, just not re-run here.
- name: Build (NVHPC)
if: matrix.nvhpc && matrix.target == 'cpu'
if: matrix.target == 'cpu'
run: |
docker exec nvhpc bash -c '
source /etc/nvhpc-env.sh
/bin/bash mfc.sh test -v --dry-run -j $(nproc) $MFC_NVHPC_TEST_FLAGS
/bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all $MFC_NVHPC_TEST_FLAGS
'

- name: Build (NVHPC GPU)
if: matrix.nvhpc && matrix.target == 'gpu'
if: matrix.target == 'gpu'
run: |
docker exec nvhpc bash -c '
source /etc/nvhpc-env.sh
Expand All @@ -330,7 +342,7 @@ jobs:
# case-optimized 2D build elides that code entirely. OpenACC only,
# since the two-pass IPO is disabled for OpenMP offload.
- name: Build (NVHPC GPU, case-optimized)
if: matrix.nvhpc && matrix.target == 'gpu'
if: matrix.target == 'gpu'
run: |
docker exec nvhpc bash -c '
source /etc/nvhpc-env.sh
Expand All @@ -339,7 +351,7 @@ jobs:
'

- name: Test (NVHPC)
if: matrix.nvhpc && matrix.target == 'cpu'
if: matrix.target == 'cpu'
run: |
docker exec nvhpc bash -c '
source /etc/nvhpc-env.sh
Expand All @@ -349,7 +361,7 @@ jobs:

# ── Cleanup ─────────────────────────────────────────────────────────
- name: Stop NVHPC container
if: always() && matrix.nvhpc
if: always()
run: docker rm -f nvhpc || true

self:
Expand Down
Loading