Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ENH: meson: implement BLAS/LAPACK auto-detection and many CI jobs #24899

Merged
merged 6 commits into from
Oct 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
284 changes: 283 additions & 1 deletion .github/workflows/linux_meson.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,36 @@
name: Test Meson build (Linux)
name: BLAS tests (Linux)

# This file is meant for testing different BLAS/LAPACK flavors and build
# options on Linux. All other yml files for Linux will only test without BLAS
# (mostly because that's easier and faster to build) or with the same 64-bit
# OpenBLAS build that is used in the wheel jobs.
#
# Jobs and their purpose:
#
# - openblas32_stable_nightly:
# Uses the 32-bit OpenBLAS builds, both the latest stable release
# and a nightly build.
# - openblas_no_pkgconfig_fedora:
# Test OpenBLAS on Fedora. Fedora doesn't ship .pc files for OpenBLAS,
# hence this exercises the "system dependency" detection method.
# - flexiblas_fedora:
# Tests FlexiBLAS (the default on Fedora for its own packages), via
# pkg-config. FlexiBLAS allows runtime switching of BLAS/LAPACK
# libraries, which is a useful capability (not tested in this job).
# - openblas_cmake:
# Tests whether OpenBLAS LP64 is detected correctly when only CMake
# and not pkg-config is installed.
# - netlib:
# Installs vanilla blas/lapack, which is the last option tried in
# auto-detection.
# - mkl:
# Tests MKL installed from PyPI (because easiest/fastest, if broken) in
# 3 ways: both LP64 and ILP64 via pkg-config, and then using the
# Single Dynamic Library (SDL, or `libmkl_rt`).
# - blis:
# Simple test for LP64 via pkg-config
# - atlas:
# Simple test for LP64 via pkg-config

on:
pull_request:
Expand Down Expand Up @@ -71,3 +103,253 @@ jobs:
export NPY_RUN_MYPY_IN_TESTSUITE=1
pip install pytest pytest-xdist hypothesis typing_extensions
spin test -j auto


openblas_no_pkgconfig_fedora:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
container: fedora:39
name: "OpenBLAS (Fedora, no pkg-config, LP64/ILP64)"
steps:
- name: Install system dependencies
run: |
dnf install git gcc-gfortran g++ python3-devel openblas-devel -y

- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
submodules: recursive
fetch-depth: 0

- name: Install dependencies
run: |
pip install -r build_requirements.txt
pip install pytest hypothesis typing_extensions

- name: Build (LP64)
run: spin build -- -Dblas=openblas -Dlapack=openblas -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg

- name: Build (ILP64)
run: |
rm -rf build
spin build -- -Duse-ilp64=true -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg


flexiblas_fedora:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
container: fedora:39
name: "FlexiBLAS (LP64, ILP64 on Fedora)"
steps:
- name: Install system dependencies
run: |
dnf install git gcc-gfortran g++ python3-devel flexiblas-devel -y

- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
submodules: recursive
fetch-depth: 0

- name: Install dependencies
run: |
pip install -r build_requirements.txt
pip install pytest hypothesis typing_extensions

- name: Build
run: spin build -- -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg

- name: Build (ILP64)
run: |
rm -rf build
spin build -- -Ddisable-optimization=true -Duse-ilp64=true

- name: Test (ILP64)
run: spin test -- numpy/linalg


openblas_cmake:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
name: "OpenBLAS with CMake"
steps:
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'

- name: Install dependencies
run: |
pip install -r build_requirements.txt
pip install pytest pytest-xdist hypothesis typing_extensions
sudo apt-get install libopenblas-dev cmake
sudo apt-get remove pkg-config

- name: Build
run: spin build -- -Ddisable-optimization=true

- name: Test
run: spin test -j auto -- numpy/linalg


netlib:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
name: "Netlib BLAS/LAPACK"
steps:
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'

- name: Install dependencies
run: |
pip install -r build_requirements.txt
sudo apt-get install liblapack-dev pkg-config

- name: Build
run: |
spin build -- -Ddisable-optimization=true

- name: Test
run: |
pip install pytest pytest-xdist hypothesis typing_extensions
spin test -j auto -- numpy/linalg


mkl:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
name: "MKL (LP64, ILP64, SDL)"
steps:
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'

- name: Install dependencies
run: |
pip install -r build_requirements.txt
pip install pytest pytest-xdist hypothesis typing_extensions
pip install mkl mkl-devel

- name: Repair MKL pkg-config files and symlinks
run: |
# MKL 2023.2 works when installed from conda-forge (except for `-iomp`
# and `-tbb` pkg-config files), Spack, or with the standalone Intel
# installer. The standalone installer is the worst option, since it's
# large and clumsy to install and requires running a setvars.sh script
# before things work. The PyPI MKL packages are broken and need the
# fixes in this step. For details, see
# https://github.com/conda-forge/intel_repack-feedstock/issues/34
cd $Python3_ROOT_DIR/lib/pkgconfig
sed -i 's/\/intel64//g' mkl*.pc
# add the expected .so -> .so.2 symlinks to fix linking
cd ..
for i in $( ls libmkl*.so.2 ); do ln -s $i ${i%.*}; done

- name: Build with defaults (LP64)
run: |
pkg-config --libs mkl-dynamic-lp64-seq # check link flags
spin build -- -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg

- name: Build with ILP64
run: |
git clean -xdf > /dev/null
pkg-config --libs mkl-dynamic-ilp64-seq
spin build -- -Duse-ilp64=true -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg

- name: Build without pkg-config (default options, SDL)
run: |
git clean -xdf > /dev/null
pushd $Python3_ROOT_DIR/lib/pkgconfig
rm mkl*.pc
popd
export MKLROOT=$Python3_ROOT_DIR
spin build -- -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg

blis:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
name: "BLIS"
steps:
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'

- name: Install dependencies
run: |
pip install -r build_requirements.txt
pip install pytest pytest-xdist hypothesis typing_extensions
sudo apt-get install libblis-dev libopenblas-dev pkg-config

- name: Add BLIS pkg-config file
run: |
# Needed because blis.pc missing in Debian:
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989076
# The alternative here would be to use another distro or Miniforge
sudo cp tools/ci/_blis_debian.pc /usr/lib/x86_64-linux-gnu/pkgconfig/blis.pc
# Check if the patch works:
pkg-config --libs blis
pkg-config --cflags blis

- name: Build
run: spin build -- -Dblas=blis -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg

atlas:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
name: "ATLAS"
steps:
- uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'

- name: Install dependencies
run: |
pip install -r build_requirements.txt
pip install pytest pytest-xdist hypothesis typing_extensions
sudo apt-get install libatlas-base-dev pkg-config

- name: Build
run: spin build -- -Dblas=blas-atlas -Dlapack=lapack-atlas -Ddisable-optimization=true

- name: Test
run: spin test -- numpy/linalg

20 changes: 11 additions & 9 deletions .github/workflows/linux_musl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,20 +48,22 @@ jobs:

ln -s /usr/local/bin/python3.10 /usr/local/bin/python

- name: test musllinux_x86_64
- name: test-musllinux_x86_64
env:
PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
run: |
python -m venv test_env
source test_env/bin/activate

# required for figuring out the system tags in openblas_support
pip install packaging

# install openblas by co-opting the CIBW setup script
RUNNER_OS=Linux sh tools/wheels/cibw_before_build.sh .
pip install scipy-openblas64

pip install -r build_requirements.txt
pip install pytest pytest-xdist hypothesis typing_extensions
pip install -r build_requirements.txt -r test_requirements.txt

# use meson to build and test
spin build
spin build --with-scipy-openblas=64 -- -Duse-ilp64=true
spin test -j auto

- name: Meson Log
shell: bash
run: |
cat build/meson-logs/meson-log.txt
19 changes: 10 additions & 9 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ jobs:
ccache -s

accelerate:
name: Accelerate ILP64
name: Accelerate (LP64, ILP64)
if: "github.repository == 'numpy/numpy'"
runs-on: macos-13
steps:
Expand All @@ -122,14 +122,15 @@ jobs:
pip install -r build_requirements.txt
pip install pytest pytest-xdist hypothesis

- name: Build NumPy against Accelerate (ILP64)
run: |
spin build -- -Dblas=accelerate -Dlapack=accelerate -Duse-ilp64=true
- name: Build against Accelerate (LP64)
run: spin build -- -Ddisable-optimization=true

- name: Show meson-log.txt
if: always()
run: 'cat build/meson-logs/meson-log.txt'
- name: Test (linalg only)
run: spin test -j2 -- numpy/linalg

- name: Test
- name: Build NumPy against Accelerate (ILP64)
run: |
spin test -j2
spin build -- -Duse-ilp64=true

- name: Test (fast tests)
run: spin test -j2