ENH: meson: implement BLAS/LAPACK auto-detection and many CI jobs #3710
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: BLAS tests (Linux) | |
# This file is meant for testing different BLAS/LAPACK flavors and build | |
# options on Linux. All other yml files for Linux will only test without BLAS | |
# (mostly because that's easier and faster to build) or with the same 64-bit | |
# OpenBLAS build that is used in the wheel jobs. | |
# | |
# Jobs and their purpose: | |
# | |
# - openblas32_stable_nightly: | |
# Uses the 32-bit OpenBLAS builds, both the latest stable release | |
# and a nightly build. | |
# - openblas_no_pkgconfig_fedora: | |
# Test OpenBLAS on Fedora. Fedora doesn't ship .pc files for OpenBLAS, | |
# hence this exercises the "system dependency" detection method. | |
# - flexiblas_fedora: | |
# Tests FlexiBLAS (the default on Fedora for its own packages), via | |
# pkg-config. FlexiBLAS allows runtime switching of BLAS/LAPACK | |
# libraries, which is a useful capability (not tested in this job). | |
# - openblas_cmake: | |
# Tests whether OpenBLAS LP64 is detected correctly when only CMake | |
# and not pkg-config is installed. | |
# - netlib: | |
# Installs vanilla blas/lapack, which is the last option tried in | |
# auto-detection. | |
# - mkl: | |
# Tests MKL installed from PyPI (because easiest/fastest, if broken) in | |
# 3 ways: both LP64 and ILP64 via pkg-config, and then using the | |
# Single Dynamic Library (SDL, or `libmkl_rt`). | |
# - blis: | |
# Simple test for LP64 via pkg-config | |
# - atlas: | |
# Simple test for LP64 via pkg-config | |
on: | |
pull_request: | |
branches: | |
- main | |
- maintenance/** | |
defaults: | |
run: | |
shell: bash | |
env: | |
PYTHON_VERSION: 3.11 | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
cancel-in-progress: true | |
permissions: | |
contents: read # to fetch code (actions/checkout) | |
jobs: | |
meson_spin: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
USE_NIGHTLY_OPENBLAS: [false, true] | |
env: | |
USE_NIGHTLY_OPENBLAS: ${{ matrix.USE_NIGHTLY_OPENBLAS }} | |
name: "Test Linux (${{ matrix.USE_NIGHTLY_OPENBLAS && 'nightly' || 'stable' }} OpenBLAS)" | |
steps: | |
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 | |
with: | |
python-version: ${{ env.PYTHON_VERSION }} | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
# Install OpenBLAS | |
set -xe | |
if [[ $USE_NIGHTLY_OPENBLAS == "true" ]]; then | |
target=$(python tools/openblas_support.py --nightly) | |
else | |
target=$(python tools/openblas_support.py) | |
fi | |
sudo cp -r $target/lib/* /usr/lib | |
sudo cp $target/include/* /usr/include | |
- name: Build | |
shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' | |
env: | |
TERM: xterm-256color | |
run: | |
spin build -- --werror | |
- name: Check build-internal dependencies | |
run: | |
ninja -C build -t missingdeps | |
- name: Check installed test and stub files | |
run: | |
python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy') | |
- name: Test | |
shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' | |
env: | |
TERM: xterm-256color | |
LD_LIBRARY_PATH: "/usr/local/lib/" # to find libopenblas.so.0 | |
run: | | |
export NPY_RUN_MYPY_IN_TESTSUITE=1 | |
pip install pytest pytest-xdist hypothesis typing_extensions | |
spin test -j auto | |
openblas_no_pkgconfig_fedora: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
container: fedora:39 | |
name: "OpenBLAS (Fedora, no pkg-config, LP64/ILP64)" | |
steps: | |
- name: Install system dependencies | |
run: | | |
dnf install git gcc-gfortran g++ python3-devel openblas-devel -y | |
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
pip install pytest hypothesis typing_extensions | |
- name: Build (LP64) | |
run: spin build -- -Dblas=openblas -Dlapack=openblas -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |
- name: Build (ILP64) | |
run: | | |
rm -rf build | |
spin build -- -Duse-ilp64=true -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |
flexiblas_fedora: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
container: fedora:39 | |
name: "FlexiBLAS (LP64, ILP64 on Fedora)" | |
steps: | |
- name: Install system dependencies | |
run: | | |
dnf install git gcc-gfortran g++ python3-devel flexiblas-devel -y | |
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
pip install pytest hypothesis typing_extensions | |
- name: Build | |
run: spin build -- -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |
- name: Build (ILP64) | |
run: | | |
rm -rf build | |
spin build -- -Ddisable-optimization=true -Duse-ilp64=true | |
- name: Test (ILP64) | |
run: spin test -- numpy/linalg | |
openblas_cmake: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
name: "OpenBLAS with CMake" | |
steps: | |
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
pip install pytest pytest-xdist hypothesis typing_extensions | |
sudo apt-get install libopenblas-dev cmake | |
sudo apt-get remove pkg-config | |
- name: Build | |
run: spin build -- -Ddisable-optimization=true | |
- name: Test | |
run: spin test -j auto -- numpy/linalg | |
netlib: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
name: "Netlib BLAS/LAPACK" | |
steps: | |
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
sudo apt-get install liblapack-dev pkg-config | |
- name: Build | |
run: | | |
spin build -- -Ddisable-optimization=true | |
- name: Test | |
run: | | |
pip install pytest pytest-xdist hypothesis typing_extensions | |
spin test -j auto -- numpy/linalg | |
mkl: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
name: "MKL (LP64, ILP64, SDL)" | |
steps: | |
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
pip install pytest pytest-xdist hypothesis typing_extensions | |
pip install mkl mkl-devel | |
- name: Repair MKL pkg-config files and symlinks | |
run: | | |
# MKL 2023.2 works when installed from conda-forge (except for `-iomp` | |
# and `-tbb` pkg-config files), Spack, or with the standalone Intel | |
# installer. The standalone installer is the worst option, since it's | |
# large and clumsy to install and requires running a setvars.sh script | |
# before things work. The PyPI MKL packages are broken and need the | |
# fixes in this step. For details, see | |
# https://github.com/conda-forge/intel_repack-feedstock/issues/34 | |
cd $Python3_ROOT_DIR/lib/pkgconfig | |
sed -i 's/\/intel64//g' mkl*.pc | |
# add the expected .so -> .so.2 symlinks to fix linking | |
cd .. | |
for i in $( ls libmkl*.so.2 ); do ln -s $i ${i%.*}; done | |
- name: Build with defaults (LP64) | |
run: | | |
pkg-config --libs mkl-dynamic-lp64-seq # check link flags | |
spin build -- -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |
- name: Build with ILP64 | |
run: | | |
git clean -xdf > /dev/null | |
pkg-config --libs mkl-dynamic-ilp64-seq | |
spin build -- -Duse-ilp64=true -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |
- name: Build without pkg-config (default options, SDL) | |
run: | | |
git clean -xdf > /dev/null | |
pushd $Python3_ROOT_DIR/lib/pkgconfig | |
rm mkl*.pc | |
popd | |
export MKLROOT=$Python3_ROOT_DIR | |
spin build -- -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |
blis: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
name: "BLIS" | |
steps: | |
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
pip install pytest pytest-xdist hypothesis typing_extensions | |
sudo apt-get install libblis-dev libopenblas-dev pkg-config | |
- name: Add BLIS pkg-config file | |
run: | | |
# Needed because blis.pc missing in Debian: | |
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989076 | |
# The alternative here would be to use another distro or Miniforge | |
sudo cp tools/ci/_blis_debian.pc /usr/lib/x86_64-linux-gnu/pkgconfig/blis.pc | |
# Check if the patch works: | |
pkg-config --libs blis | |
pkg-config --cflags blis | |
- name: Build | |
run: spin build -- -Dblas=blis -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |
atlas: | |
if: "github.repository == 'numpy/numpy'" | |
runs-on: ubuntu-latest | |
name: "ATLAS" | |
steps: | |
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 | |
with: | |
submodules: recursive | |
fetch-depth: 0 | |
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: | | |
pip install -r build_requirements.txt | |
pip install pytest pytest-xdist hypothesis typing_extensions | |
sudo apt-get install libatlas-base-dev pkg-config | |
- name: Build | |
run: spin build -- -Dblas=blas-atlas -Dlapack=lapack-atlas -Ddisable-optimization=true | |
- name: Test | |
run: spin test -- numpy/linalg | |