From 60f54962aee90389bb86bb6327eb927d73841695 Mon Sep 17 00:00:00 2001 From: ffgan Date: Sun, 19 Oct 2025 16:51:37 +0800 Subject: [PATCH 1/9] Use cibuildwheel to replace multibuild Co-authored by: nijincheng@iscas.ac.cn; --- .github/workflows/posix.yml | 142 +++++++++----------------- .gitmodules | 6 -- build-openblas.sh | 97 ++++++++++++++++++ ci-before-build.sh | 193 ++++++++++++++++++++++++++++++++++++ ci-repair-wheel.sh | 35 +++++++ ci-test.sh | 22 ++++ gfortran-install | 1 - multibuild | 1 - pyproject.toml | 17 ++++ setup.py | 18 +++- tools/build_steps.sh | 51 +++++++--- tools/build_wheel.sh | 109 -------------------- tools/docker_build_wrap.sh | 2 +- tools/gfortran_utils.sh | 183 ++++++++++++++++++++++++++++++++++ 14 files changed, 645 insertions(+), 232 deletions(-) create mode 100755 build-openblas.sh create mode 100755 ci-before-build.sh create mode 100755 ci-repair-wheel.sh create mode 100755 ci-test.sh delete mode 160000 gfortran-install delete mode 160000 multibuild create mode 100644 tools/gfortran_utils.sh diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 2943422c..88f55c96 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -23,106 +23,55 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - PLAT: [i686, x86_64] - INTERFACE64: ['0', '1'] - MB_ML_VER: ['2014'] - MB_ML_LIBC: ['manylinux'] include: - - os: macos-latest - PLAT: arm64 - INTERFACE64: '1' - - os: macos-latest - PLAT: arm64 - INTERFACE64: '0' - - os: ubuntu-latest - PLAT: x86_64 - INTERFACE64: '1' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - os: ubuntu-latest - PLAT: x86_64 - INTERFACE64: '0' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '0' - MB_ML_VER: '2014' - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '1' - MB_ML_VER: '2014' - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '0' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '1' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - exclude: - - PLAT: i686 - os: macos-latest - - PLAT: i686 - INTERFACE64: '1' + - { os: ubuntu-latest, PLAT: i686, INTERFACE64: '0', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '0', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '1', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + + - { os: macos-latest, PLAT: x86_64, INTERFACE64: '0', MB_ML_LIBC: macosx} + - { os: macos-latest, PLAT: x86_64, INTERFACE64: '1', MB_ML_LIBC: macosx} + + - { os: macos-latest, PLAT: arm64, INTERFACE64: '0', MB_ML_LIBC: macosx} + - { os: macos-latest, PLAT: arm64, INTERFACE64: '1', MB_ML_LIBC: macosx} + + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '0', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '1', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '0', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '1', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '0', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '1', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + env: NIGHTLY: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - MB_PYTHON_VERSION: ${{ matrix.python-version }} - TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} MB_ML_LIBC: ${{ matrix.MB_ML_LIBC }} MB_ML_VER: ${{ matrix.MB_ML_VER }} INTERFACE64: ${{ matrix.INTERFACE64 }} BUILD_DIR: ${{ github.workspace }} PLAT: ${{ matrix.PLAT }} + OS-NAME: ${{ matrix.os }} steps: - uses: actions/checkout@v4.1.1 with: submodules: recursive fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.9 + - name: Set extra env run: | echo "DOCKER_TEST_IMAGE=$(echo multibuild/xenial_${{ matrix.PLAT}})" >> $GITHUB_ENV; - uses: maxim-lobanov/setup-xcode@v1.6.0 - if: ${{ matrix.os == 'macos-latest' }} + if: ${{ contains(matrix.os, 'macos') }} with: - xcode-version: '15.4' + xcode-version: '16.0' - name: Print some Environment variable run: | echo "PLAT: ${PLAT}" echo "DOCKER_TEST_IMAGE: ${DOCKER_TEST_IMAGE}" - - name: Install VirtualEnv - run: | - python3 -m pip install --upgrade pip - pip install virtualenv - - name: Build OpenBLAS - run: | - set -xeo pipefail - source tools/build_steps.sh - echo "------ BEFORE BUILD ---------" - before_build - if [[ "$NIGHTLY" = "true" ]]; then - echo "------ CLEAN CODE --------" - clean_code $REPO_DIR develop - echo "------ BUILD LIB --------" - build_lib "$PLAT" "$INTERFACE64" "1" - else - echo "------ CLEAN CODE --------" - clean_code $REPO_DIR $OPENBLAS_COMMIT - echo "------ BUILD LIB --------" - build_lib "$PLAT" "$INTERFACE64" "0" - fi # - name: Setup tmate session # if: ${{ failure() }} @@ -130,34 +79,33 @@ jobs: # with: # limit-access-to-actor: true - - name: Build and test wheel - run: | - if [[ "$NIGHTLY" = "true" ]]; then - # Set the pyproject.toml version: convert v0.3.24-30-g138ed79f to 0.3.34.30 - version=$(cd OpenBLAS && git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") - sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml - fi - if [ "macos-latest" == "${{ matrix.os }}" ]; then - source tools/build_wheel.sh - else - libc=${MB_ML_LIBC:-manylinux} - docker_image=quay.io/pypa/${libc}${MB_ML_VER}_${PLAT} - docker run --rm -e INTERFACE64="${INTERFACE64}" \ - -e MB_ML_LIBC="${MB_ML_LIBC}" \ - -v $(pwd):/openblas $docker_image \ - /bin/bash -xe /openblas/tools/build_wheel.sh - sudo chmod -R a+w dist - fi - - - uses: actions/upload-artifact@v4.3.0 + - name: Build and Test wheels + uses: pypa/cibuildwheel@v3.1.4 + with: + output-dir: dist + env: + CIBW_ARCHS: ${{matrix.PLAT}} + CIBW_BUILD_VERBOSITY: 1 + CIBW_BUILD: "cp39-${{ matrix.MB_ML_LIBC }}_${{matrix.PLAT}}" + + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MUSLLINUX_X86_64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + + - name: Upload wheels to artifacts + uses: actions/upload-artifact@v4.3.0 with: name: wheels-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }}-${{ matrix.MB_ML_VER }} path: dist/scipy_openblas*.whl - - uses: actions/upload-artifact@v4.3.0 + + - name: Upload openblas to artifacts + uses: actions/upload-artifact@v4.3.0 with: name: openblas-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }}-${{ matrix.MB_ML_VER }} - path: libs/openblas*.tar.gz + path: dist/openblas*.tar.gz - uses: conda-incubator/setup-miniconda@v3.2.0 with: diff --git a/.gitmodules b/.gitmodules index 9a0ddc0f..546361d9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,3 @@ [submodule "OpenBLAS"] path = OpenBLAS url = https://github.com/xianyi/OpenBLAS.git -[submodule "multibuild"] - path = multibuild - url = https://github.com/multi-build/multibuild.git -[submodule "gfortran-install"] - path = gfortran-install - url = https://github.com/MacPython/gfortran-install.git diff --git a/build-openblas.sh b/build-openblas.sh new file mode 100755 index 00000000..79d4f25e --- /dev/null +++ b/build-openblas.sh @@ -0,0 +1,97 @@ +#! /bin/bash + + +# Most of the content in this file comes from https://github.com/multi-build/multibuild, with some modifications +# Follow the license below + + + +# .. _license: + +# ********************* +# Copyright and License +# ********************* + +# The multibuild package, including all examples, code snippets and attached +# documentation is covered by the 2-clause BSD license. + +# Copyright (c) 2013-2024, Matt Terry and Matthew Brett; all rights +# reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +set -xeo pipefail +source tools/build_steps.sh +echo "------ BEFORE BUILD ---------" +before_build + +function fill_submodule { + # Restores .git directory to submodule, if necessary + # See: + # https://stackoverflow.com/questions/41776331/is-there-a-way-to-reconstruct-a-git-directory-for-a-submodule + local repo_dir="$1" + [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 + local git_loc="$repo_dir/.git" + # For ordinary submodule, .git is a file. + [ -d "$git_loc" ] && return + # Need to recreate .git directory for submodule + local origin_url=$(cd "$repo_dir" && git config --get remote.origin.url) + local repo_copy="$repo_dir-$RANDOM" + git clone --recursive "$repo_dir" "$repo_copy" + rm -rf "$repo_dir" + mv "${repo_copy}" "$repo_dir" + (cd "$repo_dir" && git remote set-url origin $origin_url) +} + +function clean_code { + local repo_dir=${1:-$REPO_DIR} + local build_commit=${2:-$BUILD_COMMIT} + [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 + [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 + # The package $repo_dir may be a submodule. git submodules do not + # have a .git directory. If $repo_dir is copied around, tools like + # Versioneer which require that it be a git repository are unable + # to determine the version. Give submodule proper git directory + fill_submodule "$repo_dir" + (cd $repo_dir \ + && git fetch origin --tags \ + && git checkout $build_commit \ + && git clean -fxd \ + && git reset --hard \ + && git submodule update --init --recursive) +} + + +if [[ "$NIGHTLY" = "true" ]]; then + echo "------ CLEAN CODE --------" + clean_code $REPO_DIR develop + echo "------ BUILD LIB --------" + build_lib "$PLAT" "$INTERFACE64" "1" +else + echo "------ CLEAN CODE --------" + clean_code $REPO_DIR $OPENBLAS_COMMIT + echo "------ BUILD LIB --------" + build_lib "$PLAT" "$INTERFACE64" "0" +fi \ No newline at end of file diff --git a/ci-before-build.sh b/ci-before-build.sh new file mode 100755 index 00000000..6ac38aa1 --- /dev/null +++ b/ci-before-build.sh @@ -0,0 +1,193 @@ +#! /bin/bash + + +# Most of the content in this file comes from https://github.com/multi-build/multibuild, with some modifications +# Follow the license below + + + +# .. _license: + +# ********************* +# Copyright and License +# ********************* + +# The multibuild package, including all examples, code snippets and attached +# documentation is covered by the 2-clause BSD license. + +# Copyright (c) 2013-2024, Matt Terry and Matthew Brett; all rights +# reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +#! /bin/bash +set -xe + +if [[ "$NIGHTLY" = "true" ]]; then + # Set the pyproject.toml version: convert v0.3.24-30-g138ed79f to 0.3.34.30 + version=$(cd OpenBLAS && git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") + sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml +fi + + +#!/bin/bash +# Utilities for both OSX and Docker Linux +# python or python3 should be on the PATH + +# Only source common_utils once +if [ -n "$COMMON_UTILS_SOURCED" ]; then + return +fi +COMMON_UTILS_SOURCED=1 + +# Turn on exit-if-error +set -e + +MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") +DOWNLOADS_SDIR=downloads +PYPY_URL=https://downloads.python.org/pypy +# For back-compatibility. We use the "ensurepip" module now +# instead of get-pip.py +GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py + +# Unicode width, default 32. Used here and in travis_linux_steps.sh +# In docker_build_wrap.sh it is passed in when calling "docker run" +# The docker test images also use it when choosing the python to run +# with, so it is passed in when calling "docker run" for tests. +UNICODE_WIDTH=${UNICODE_WIDTH:-32} + +if [ $(uname) == "Darwin" ]; then + IS_MACOS=1; IS_OSX=1; +else + # In the manylinux_2_24 image, based on Debian9, "python" is not installed + # so link in something for the various system calls before PYTHON_EXE is set + which python || export PATH=/opt/python/cp39-cp39/bin:$PATH + + if [ "$MB_ML_LIBC" == "musllinux" ]; then + IS_ALPINE=1; + MB_ML_VER=${MB_ML_VER:-"_1_2"} + else + # Default Manylinux version + MB_ML_VER=${MB_ML_VER:-2014} + fi +fi + +# Work round bug in travis xcode image described at +# https://github.com/direnv/direnv/issues/210 +shell_session_update() { :; } + +# Workaround for https://github.com/travis-ci/travis-ci/issues/8703 +# suggested by Thomas K at +# https://github.com/travis-ci/travis-ci/issues/8703#issuecomment-347881274 +unset -f cd +unset -f pushd +unset -f popd + +function cmd_notexit { + # wraps a command, capturing its return code and preventing it + # from exiting the shell. Handles -e / +e modes. + # Parameters + # cmd - command + # any further parameters are passed to the wrapped command + # If called without an argument, it will exit the shell with an error + local cmd=$1 + if [ -z "$cmd" ];then echo "no command"; exit 1; fi + if [[ $- = *e* ]]; then errexit_set=true; fi + set +e + ("${@:1}") ; retval=$? + [[ -n $errexit_set ]] && set -e + return $retval +} + +# Build OpenBLAS +source build-openblas.sh + + +ls libs/openblas* >/dev/null 2>&1 && true +if [ "$?" != "0" ]; then + # inside docker + cd /project +fi +PYTHON=${PYTHON:-python3.9} + +mkdir -p local/openblas +mkdir -p dist +$PYTHON -m pip install wheel auditwheel + +# This will fail if there is more than one file in libs +tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas*.tar.gz + +# do not package the static libs and symlinks, only take the shared object +find local/scipy_openblas64/lib -maxdepth 1 -type l -delete +rm local/scipy_openblas64/lib/*.a +# Check that the pyproject.toml and the pkgconfig versions agree. +py_version=$(grep "^version" pyproject.toml | sed -e "s/version = \"//") +pkg_version=$(grep "version=" ./local/scipy_openblas64/lib/pkgconfig/scipy-openblas*.pc | sed -e "s/version=//" | sed -e "s/dev//") +if [[ -z "$pkg_version" ]]; then + echo Could not read version from pkgconfig file + exit 1 +fi +if [[ $py_version != $pkg_version* ]]; then + echo Version from pyproject.toml "$py_version" does not match version from build "pkg_version" + exit 1 +fi + +if [ $(uname) == "Darwin" ]; then + soname=$(cd local/scipy_openblas64/lib; ls libscipy_openblas*.dylib) + echo otool -D local/scipy_openblas64/lib/$soname + otool -D local/scipy_openblas64/lib/$soname + # issue 153: there is a ".0" in the install_name. Remove it + # also add a @rpath + install_name_tool -id @rpath/$soname local/scipy_openblas64/lib/$soname +fi + +rm -rf local/scipy_openblas64/lib/pkgconfig +echo "" >> LICENSE.txt +echo "----" >> LICENSE.txt +echo "" >> LICENSE.txt +if [ $(uname) == "Darwin" ]; then + cat tools/LICENSE_osx.txt >> LICENSE.txt +else + cat tools/LICENSE_linux.txt >> LICENSE.txt +fi + +if [ "${INTERFACE64}" != "1" ]; then + # rewrite the name of the project to scipy-openblas32 + # this is a hack, but apparently there is no other way to change the name + # of a pyproject.toml project + # + # use the BSD variant of sed -i and remove the backup + sed -e "s/openblas64/openblas32/" -i.bak pyproject.toml + rm *.bak + mv local/scipy_openblas64 local/scipy_openblas32 + sed -e "s/openblas_get_config64_/openblas_get_config/" -i.bak local/scipy_openblas32/__init__.py + sed -e "s/cflags =.*/cflags = '-DBLAS_SYMBOL_PREFIX=scipy_'/" -i.bak local/scipy_openblas32/__init__.py + sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__main__.py + sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__init__.py + rm local/scipy_openblas32/*.bak +fi + +rm -rf dist/* diff --git a/ci-repair-wheel.sh b/ci-repair-wheel.sh new file mode 100755 index 00000000..22fb2bac --- /dev/null +++ b/ci-repair-wheel.sh @@ -0,0 +1,35 @@ +#! /bin/bash +set -xe + +PYTHON=${PYTHON:-python3.9} + +if [ $(uname) == "Darwin" ]; then + $PYTHON -m pip install delocate + # move the mis-named scipy_openblas64-none-any.whl to a platform-specific name + # if [ "${PLAT}" == "arm64" ]; then + # for f in $2/*.whl; do mv $f "${f/%any.whl/macosx_11_0_$PLAT.whl}"; done + # else + # for f in $2/*.whl; do mv $f "${f/%any.whl/macosx_10_9_$PLAT.whl}"; done + # fi + delocate-wheel -w $1 -v $2 + + cp libs/openblas*.tar.gz dist/ +else + auditwheel repair -w $1 --lib-sdir /lib $2 + # rm dist/scipy_openblas*-none-any.whl + # rm {dest_dir}/*.whl + + # Add an RPATH to libgfortran: + # https://github.com/pypa/auditwheel/issues/451 + if [ "$MB_ML_LIBC" == "musllinux" ]; then + apk add zip + else + yum install -y zip + fi + unzip $1/*.whl "*libgfortran*" + patchelf --force-rpath --set-rpath '$ORIGIN' */lib/libgfortran* + zip $1/*.whl */lib/libgfortran* + mkdir -p /output + # copy libs/openblas*.tar.gz to dist/ + cp libs/openblas*.tar.gz /output/ +fi diff --git a/ci-test.sh b/ci-test.sh new file mode 100755 index 00000000..1a553636 --- /dev/null +++ b/ci-test.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Test that the wheel works with a different python +set -xe + +if [ "${PLAT}" == "arm64" ]; then + # Cannot test + exit 0 +fi + +PYTHON=python3.9 +if [ "$(uname)" == "Darwin" -a "${PLAT}" == "x86_64" ]; then + which python3.9 + PYTHON="arch -x86_64 python3.9" +fi +if [ "${INTERFACE64}" != "1" ]; then + # cibuildwheel will install the wheel automatically + # $PYTHON -m pip install --no-index --find-links /tmp/cibuildwheel/repaired_wheel scipy_openblas32 + $PYTHON -m scipy_openblas32 +else + # $PYTHON -m pip install --no-index --find-links /tmp/cibuildwheel/repaired_wheel scipy_openblas64 + $PYTHON -m scipy_openblas64 +fi diff --git a/gfortran-install b/gfortran-install deleted file mode 160000 index 3dd38d9c..00000000 --- a/gfortran-install +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3dd38d9ce78b3890598cb0eff18a7bec50c06f5e diff --git a/multibuild b/multibuild deleted file mode 160000 index 24f1446f..00000000 --- a/multibuild +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 24f1446f6477893b2ef9d721aabcee7a494ca2b1 diff --git a/pyproject.toml b/pyproject.toml index 90bc7f9c..61a4a055 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,3 +39,20 @@ install_requires = "importlib-metadata ~= 1.0 ; python_version < '3.8'" [tool.setuptools.package-data] scipy_openblas64 = ["lib/*", "include/*", "lib/pkgconfig/*", "lib/cmake/openblas/*"] + +[tool.cibuildwheel] +before-build = "bash ci-before-build.sh" +repair-wheel-command = "bash ci-repair-wheel.sh {dest_dir} {wheel}" +test-command = "cd {package} && bash ci-test.sh " +environment-pass = [ + "REPO_DIR", + "OPENBLAS_COMMIT", + "MACOSX_DEPLOYMENT_TARGET", + "NIGHTLY", + "MB_ML_LIBC", + "MB_ML_VER", + "INTERFACE64", + "BUILD_DIR", + "PLAT", + "OS-NAME", +] diff --git a/setup.py b/setup.py index 60684932..cd0a2c06 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,19 @@ from setuptools import setup +from setuptools.dist import Distribution +from wheel.bdist_wheel import bdist_wheel as _bdist_wheel -setup() + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + +class bdist_wheel(_bdist_wheel): + def get_tag(self): + return "py3", "none", _bdist_wheel.get_tag(self)[2] + + +setup( + distclass=BinaryDistribution, + cmdclass={"bdist_wheel": bdist_wheel}, +) diff --git a/tools/build_steps.sh b/tools/build_steps.sh index f87758b1..5569eca1 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -2,10 +2,27 @@ BUILD_PREFIX=/usr/local ROOT_DIR=$(dirname $(dirname "${BASH_SOURCE[0]}")) -source ${ROOT_DIR}/multibuild/common_utils.sh MB_PYTHON_VERSION=3.9 +function any_python { + for cmd in $PYTHON_EXE python3 python; do + if [ -n "$(type -t $cmd)" ]; then + echo $cmd + return + fi + done + echo "Could not find python or python3" + exit 1 +} + +function get_os { + # Report OS as given by uname + # Use any Python that comes to hand. + $(any_python) -c 'import platform; print(platform.uname()[0])' +} + + function before_build { # Manylinux Python version set in build_lib if [ -n "$IS_OSX" ]; then @@ -19,12 +36,13 @@ function before_build { sudo chmod 777 /usr/local/include touch /usr/local/include/.dir_exists fi - source ${ROOT_DIR}/multibuild/osx_utils.sh - get_macpython_environment ${MB_PYTHON_VERSION} venv + # get_macpython_environment ${MB_PYTHON_VERSION} venv + python3.9 -m venv venv + source venv/bin/activate # Since install_fortran uses `uname -a` to determine arch, # force the architecture arch -${PLAT} bash -s << EOF -source ${ROOT_DIR}/gfortran-install/gfortran_utils.sh +source tools/gfortran_utils.sh install_gfortran EOF # Deployment target set by gfortran_utils @@ -115,18 +133,19 @@ function build_lib { # Manylinux wrapper local libc=${MB_ML_LIBC:-manylinux} local docker_image=quay.io/pypa/${libc}${manylinux}_${plat} - docker pull $docker_image - # Docker sources this script, and runs `do_build_lib` - docker run --rm \ - -e BUILD_PREFIX="$BUILD_PREFIX" \ - -e PLAT="${plat}" \ - -e INTERFACE64="${interface64}" \ - -e NIGHTLY="${nightly}" \ - -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ - -e MB_ML_VER=${manylinux} \ - -e MB_ML_LIBC=${libc} \ - -v $PWD:/io \ - $docker_image /io/tools/docker_build_wrap.sh + # docker pull $docker_image + # # Docker sources this script, and runs `do_build_lib` + # docker run --rm \ + # -e BUILD_PREFIX="$BUILD_PREFIX" \ + # -e PLAT="${plat}" \ + # -e INTERFACE64="${interface64}" \ + # -e NIGHTLY="${nightly}" \ + # -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ + # -e MB_ML_VER=${manylinux} \ + # -e MB_ML_LIBC=${libc} \ + # -v $PWD:/io \ + # $docker_image /io/tools/docker_build_wrap.sh + bash ./tools/docker_build_wrap.sh } function patch_source { diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index 776a3aea..dfd8bde5 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -2,114 +2,5 @@ # $INTERFACE64 ("1" or "0") # $PLAT (x86_64, i686, arm64, aarch64, s390x, ppc64le) - -set -xe - -ls libs/openblas* >/dev/null 2>&1 && true -if [ "$?" != "0" ]; then - # inside docker - cd /openblas -fi -PYTHON=${PYTHON:-python3.9} - -mkdir -p local/openblas -mkdir -p dist -$PYTHON -m pip install wheel auditwheel - -# This will fail if there is more than one file in libs -tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas*.tar.gz - -# do not package the static libs and symlinks, only take the shared object -find local/scipy_openblas64/lib -maxdepth 1 -type l -delete -rm local/scipy_openblas64/lib/*.a -# Check that the pyproject.toml and the pkgconfig versions agree. -py_version=$(grep "^version" pyproject.toml | sed -e "s/version = \"//") -pkg_version=$(grep "version=" ./local/scipy_openblas64/lib/pkgconfig/scipy-openblas*.pc | sed -e "s/version=//" | sed -e "s/dev//") -if [[ -z "$pkg_version" ]]; then - echo Could not read version from pkgconfig file - exit 1 -fi -if [[ $py_version != $pkg_version* ]]; then - echo Version from pyproject.toml "$py_version" does not match version from build "pkg_version" - exit 1 -fi - -if [ $(uname) == "Darwin" ]; then - soname=$(cd local/scipy_openblas64/lib; ls libscipy_openblas*.dylib) - echo otool -D local/scipy_openblas64/lib/$soname - otool -D local/scipy_openblas64/lib/$soname - # issue 153: there is a ".0" in the install_name. Remove it - # also add a @rpath - install_name_tool -id @rpath/$soname local/scipy_openblas64/lib/$soname -fi - -rm -rf local/scipy_openblas64/lib/pkgconfig -echo "" >> LICENSE.txt -echo "----" >> LICENSE.txt -echo "" >> LICENSE.txt -if [ $(uname) == "Darwin" ]; then - cat tools/LICENSE_osx.txt >> LICENSE.txt -else - cat tools/LICENSE_linux.txt >> LICENSE.txt -fi - -if [ "${INTERFACE64}" != "1" ]; then - # rewrite the name of the project to scipy-openblas32 - # this is a hack, but apparently there is no other way to change the name - # of a pyproject.toml project - # - # use the BSD variant of sed -i and remove the backup - sed -e "s/openblas64/openblas32/" -i.bak pyproject.toml - rm *.bak - mv local/scipy_openblas64 local/scipy_openblas32 - sed -e "s/openblas_get_config64_/openblas_get_config/" -i.bak local/scipy_openblas32/__init__.py - sed -e "s/cflags =.*/cflags = '-DBLAS_SYMBOL_PREFIX=scipy_'/" -i.bak local/scipy_openblas32/__init__.py - sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__main__.py - sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__init__.py - rm local/scipy_openblas32/*.bak -fi - -rm -rf dist/* $PYTHON -m pip wheel -w dist -v . -if [ $(uname) == "Darwin" ]; then - $PYTHON -m pip install delocate - # move the mis-named scipy_openblas64-none-any.whl to a platform-specific name - if [ "${PLAT}" == "arm64" ]; then - for f in dist/*.whl; do mv $f "${f/%any.whl/macosx_11_0_$PLAT.whl}"; done - else - for f in dist/*.whl; do mv $f "${f/%any.whl/macosx_10_9_$PLAT.whl}"; done - fi - delocate-wheel -v dist/*.whl -else - auditwheel repair -w dist --lib-sdir /lib dist/*.whl - rm dist/scipy_openblas*-none-any.whl - # Add an RPATH to libgfortran: - # https://github.com/pypa/auditwheel/issues/451 - if [ "$MB_ML_LIBC" == "musllinux" ]; then - apk add zip - else - yum install -y zip - fi - unzip dist/*.whl "*libgfortran*" - patchelf --force-rpath --set-rpath '$ORIGIN' */lib/libgfortran* - zip dist/*.whl */lib/libgfortran* -fi - -if [ "${PLAT}" == "arm64" ]; then - # Cannot test - exit 0 -fi -# Test that the wheel works with a different python -PYTHON=python3.11 -if [ "$(uname)" == "Darwin" -a "${PLAT}" == "x86_64" ]; then - which python3.11 - PYTHON="arch -x86_64 python3.11" -fi -if [ "${INTERFACE64}" != "1" ]; then - $PYTHON -m pip install --no-index --find-links dist scipy_openblas32 - $PYTHON -m scipy_openblas32 -else - $PYTHON -m pip install --no-index --find-links dist scipy_openblas64 - $PYTHON -m scipy_openblas64 -fi diff --git a/tools/docker_build_wrap.sh b/tools/docker_build_wrap.sh index e9f7b4be..a385950c 100755 --- a/tools/docker_build_wrap.sh +++ b/tools/docker_build_wrap.sh @@ -5,6 +5,6 @@ set -e # Change into root directory of repo -cd /io +# cd /io source tools/build_steps.sh do_build_lib "$PLAT" "" "$INTERFACE64" "$NIGHTLY" diff --git a/tools/gfortran_utils.sh b/tools/gfortran_utils.sh new file mode 100644 index 00000000..b7127f50 --- /dev/null +++ b/tools/gfortran_utils.sh @@ -0,0 +1,183 @@ +# this file come from https://github.com/MacPython/gfortran-install +# Follow the license below + + +# gfortran-install license +# Copyright 2016-2021 Matthew Brett, Isuru Fernando, Matti Picus + +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Bash utilities for use with gfortran + +ARCHIVE_SDIR="${ARCHIVE_SDIR:-archives}" + +GF_UTIL_DIR=$(dirname "${BASH_SOURCE[0]}") + +function get_distutils_platform { + # Report platform as in form of distutils get_platform. + # This is like the platform tag that pip will use. + # Modify fat architecture tags on macOS to reflect compiled architecture + + # Deprecate this function once get_distutils_platform_ex is used in all + # downstream projects + local plat=$1 + case $plat in + i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; + *) echo Did not recognize plat $plat; return 1 ;; + esac + local uname=${2:-$(uname)} + if [ "$uname" != "Darwin" ]; then + if [ "$plat" == "intel" ]; then + echo plat=intel not allowed for Manylinux + return 1 + fi + echo "manylinux1_$plat" + return + fi + # macOS 32-bit arch is i386 + [ "$plat" == "i686" ] && plat="i386" + local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) + echo "macosx_${target}_${plat}" +} + +function get_distutils_platform_ex { + # Report platform as in form of distutils get_platform. + # This is like the platform tag that pip will use. + # Modify fat architecture tags on macOS to reflect compiled architecture + # For non-darwin, report manylinux version + local plat=$1 + local mb_ml_ver=${MB_ML_VER:-1} + case $plat in + i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; + *) echo Did not recognize plat $plat; return 1 ;; + esac + local uname=${2:-$(uname)} + if [ "$uname" != "Darwin" ]; then + if [ "$plat" == "intel" ]; then + echo plat=intel not allowed for Manylinux + return 1 + fi + echo "manylinux${mb_ml_ver}_${plat}" + return + fi + # macOS 32-bit arch is i386 + [ "$plat" == "i686" ] && plat="i386" + local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) + echo "macosx_${target}_${plat}" +} + +function get_macosx_target { + # Report MACOSX_DEPLOYMENT_TARGET as given by distutils get_platform. + python3 -c "import sysconfig as s; print(s.get_config_vars()['MACOSX_DEPLOYMENT_TARGET'])" +} + +function check_gfortran { + # Check that gfortran exists on the path + if [ -z "$(which gfortran)" ]; then + echo Missing gfortran + exit 1 + fi +} + +function get_gf_lib_for_suf { + local suffix=$1 + local prefix=$2 + local plat=${3:-$PLAT} + local uname=${4:-$(uname)} + if [ -z "$prefix" ]; then echo Prefix not defined; exit 1; fi + local plat_tag=$(get_distutils_platform_ex $plat $uname) + if [ -n "$suffix" ]; then suffix="-$suffix"; fi + local fname="$prefix-${plat_tag}${suffix}.tar.gz" + local out_fname="${ARCHIVE_SDIR}/$fname" + [ -s $out_fname ] || (echo "$out_fname is empty"; exit 24) + echo "$out_fname" +} + +if [ "$(uname)" == "Darwin" ]; then + mac_target=${MACOSX_DEPLOYMENT_TARGET:-$(get_macosx_target)} + export MACOSX_DEPLOYMENT_TARGET=$mac_target + # Keep this for now as some builds might depend on this being + # available before install_gfortran is called + export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e + # Set SDKROOT env variable if not set + export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} + + function download_and_unpack_gfortran { + local arch=$1 + local type=$2 + curl -L -O https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-${arch}-${type}.tar.gz + case ${arch}-${type} in + arm64-native) + export GFORTRAN_SHA=0d5c118e5966d0fb9e7ddb49321f63cac1397ce8 + ;; + arm64-cross) + export GFORTRAN_SHA=527232845abc5af21f21ceacc46fb19c190fe804 + ;; + x86_64-native) + export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e + ;; + x86_64-cross) + export GFORTRAN_SHA=107604e57db97a0ae3e7ca7f5dd722959752f0b3 + ;; + esac + if [[ "$(shasum gfortran-darwin-${arch}-${type}.tar.gz)" != "${GFORTRAN_SHA} gfortran-darwin-${arch}-${type}.tar.gz" ]]; then + echo "shasum mismatch for gfortran-darwin-${arch}-${type}" + exit 1 + fi + sudo mkdir -p /opt/ + sudo cp "gfortran-darwin-${arch}-${type}.tar.gz" /opt/gfortran-darwin-${arch}-${type}.tar.gz + pushd /opt + sudo tar -xvf gfortran-darwin-${arch}-${type}.tar.gz + sudo rm gfortran-darwin-${arch}-${type}.tar.gz + popd + if [[ "${type}" == "native" ]]; then + # Link these into /usr/local so that there's no need to add rpath or -L + for f in libgfortran.dylib libgfortran.5.dylib libgcc_s.1.dylib libgcc_s.1.1.dylib libquadmath.dylib libquadmath.0.dylib; do + ln -sf /opt/gfortran-darwin-${arch}-${type}/lib/$f /usr/local/lib/$f + done + # Add it to PATH + ln -sf /opt/gfortran-darwin-${arch}-${type}/bin/gfortran /usr/local/bin/gfortran + fi + } + + function install_arm64_cross_gfortran { + download_and_unpack_gfortran arm64 cross + export FC_ARM64="$(find /opt/gfortran-darwin-arm64-cross/bin -name "*-gfortran")" + local libgfortran="$(find /opt/gfortran-darwin-arm64-cross/lib -name libgfortran.dylib)" + local libdir=$(dirname $libgfortran) + + export FC_ARM64_LDFLAGS="-L$libdir -Wl,-rpath,$libdir" + if [[ "${PLAT:-}" == "arm64" ]]; then + export FC=$FC_ARM64 + fi + } + function install_gfortran { + download_and_unpack_gfortran $(uname -m) native + check_gfortran + if [[ "${PLAT:-}" == "universal2" || "${PLAT:-}" == "arm64" ]]; then + install_arm64_cross_gfortran + fi + } + + function get_gf_lib { + # Get lib with gfortran suffix + get_gf_lib_for_suf "gf_${GFORTRAN_SHA:0:7}" $@ + } +else + function install_gfortran { + # No-op - already installed on manylinux image + check_gfortran + } + + function get_gf_lib { + # Get library with no suffix + get_gf_lib_for_suf "" $@ + } +fi From cce6ab505b51b86105ed8282c990d16a769b918c Mon Sep 17 00:00:00 2001 From: ffgan Date: Sun, 19 Oct 2025 20:24:12 +0800 Subject: [PATCH 2/9] incremented the wheel build number Co-authored by: nijincheng@iscas.ac.cn; --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 61a4a055..95988b76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "scipy-openblas64" # v0.3.30 -version = "0.3.30.0.2" +version = "0.3.30.0.3" requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" From e6c99f5ca6de84aa88474103e81c82f1fa4eb160 Mon Sep 17 00:00:00 2001 From: ffgan Date: Mon, 20 Oct 2025 17:07:22 +0800 Subject: [PATCH 3/9] fix travis error Co-authored by: nijincheng@iscas.ac.cn; --- ci-before-build.sh | 67 +--------------------------------------- tools/build_prepare.sh | 69 ++++++++++++++++++++++++++++++++++++++++++ tools/build_wheel.sh | 27 +++++++++++++++++ 3 files changed, 97 insertions(+), 66 deletions(-) create mode 100644 tools/build_prepare.sh diff --git a/ci-before-build.sh b/ci-before-build.sh index 6ac38aa1..a7f4d81e 100755 --- a/ci-before-build.sh +++ b/ci-before-build.sh @@ -125,69 +125,4 @@ function cmd_notexit { # Build OpenBLAS source build-openblas.sh - -ls libs/openblas* >/dev/null 2>&1 && true -if [ "$?" != "0" ]; then - # inside docker - cd /project -fi -PYTHON=${PYTHON:-python3.9} - -mkdir -p local/openblas -mkdir -p dist -$PYTHON -m pip install wheel auditwheel - -# This will fail if there is more than one file in libs -tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas*.tar.gz - -# do not package the static libs and symlinks, only take the shared object -find local/scipy_openblas64/lib -maxdepth 1 -type l -delete -rm local/scipy_openblas64/lib/*.a -# Check that the pyproject.toml and the pkgconfig versions agree. -py_version=$(grep "^version" pyproject.toml | sed -e "s/version = \"//") -pkg_version=$(grep "version=" ./local/scipy_openblas64/lib/pkgconfig/scipy-openblas*.pc | sed -e "s/version=//" | sed -e "s/dev//") -if [[ -z "$pkg_version" ]]; then - echo Could not read version from pkgconfig file - exit 1 -fi -if [[ $py_version != $pkg_version* ]]; then - echo Version from pyproject.toml "$py_version" does not match version from build "pkg_version" - exit 1 -fi - -if [ $(uname) == "Darwin" ]; then - soname=$(cd local/scipy_openblas64/lib; ls libscipy_openblas*.dylib) - echo otool -D local/scipy_openblas64/lib/$soname - otool -D local/scipy_openblas64/lib/$soname - # issue 153: there is a ".0" in the install_name. Remove it - # also add a @rpath - install_name_tool -id @rpath/$soname local/scipy_openblas64/lib/$soname -fi - -rm -rf local/scipy_openblas64/lib/pkgconfig -echo "" >> LICENSE.txt -echo "----" >> LICENSE.txt -echo "" >> LICENSE.txt -if [ $(uname) == "Darwin" ]; then - cat tools/LICENSE_osx.txt >> LICENSE.txt -else - cat tools/LICENSE_linux.txt >> LICENSE.txt -fi - -if [ "${INTERFACE64}" != "1" ]; then - # rewrite the name of the project to scipy-openblas32 - # this is a hack, but apparently there is no other way to change the name - # of a pyproject.toml project - # - # use the BSD variant of sed -i and remove the backup - sed -e "s/openblas64/openblas32/" -i.bak pyproject.toml - rm *.bak - mv local/scipy_openblas64 local/scipy_openblas32 - sed -e "s/openblas_get_config64_/openblas_get_config/" -i.bak local/scipy_openblas32/__init__.py - sed -e "s/cflags =.*/cflags = '-DBLAS_SYMBOL_PREFIX=scipy_'/" -i.bak local/scipy_openblas32/__init__.py - sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__main__.py - sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__init__.py - rm local/scipy_openblas32/*.bak -fi - -rm -rf dist/* +source tools/build_prepare.sh \ No newline at end of file diff --git a/tools/build_prepare.sh b/tools/build_prepare.sh new file mode 100644 index 00000000..0d241c38 --- /dev/null +++ b/tools/build_prepare.sh @@ -0,0 +1,69 @@ +#! /bin/bash + +set -xe + +ls libs/openblas* >/dev/null 2>&1 && true +if [ "$?" != "0" ]; then + # inside docker + cd /project +fi +PYTHON=${PYTHON:-python3.9} + +mkdir -p local/openblas +mkdir -p dist +$PYTHON -m pip install wheel auditwheel + +# This will fail if there is more than one file in libs +tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas*.tar.gz + +# do not package the static libs and symlinks, only take the shared object +find local/scipy_openblas64/lib -maxdepth 1 -type l -delete +rm local/scipy_openblas64/lib/*.a +# Check that the pyproject.toml and the pkgconfig versions agree. +py_version=$(grep "^version" pyproject.toml | sed -e "s/version = \"//") +pkg_version=$(grep "version=" ./local/scipy_openblas64/lib/pkgconfig/scipy-openblas*.pc | sed -e "s/version=//" | sed -e "s/dev//") +if [[ -z "$pkg_version" ]]; then + echo Could not read version from pkgconfig file + exit 1 +fi +if [[ $py_version != $pkg_version* ]]; then + echo Version from pyproject.toml "$py_version" does not match version from build "pkg_version" + exit 1 +fi + +if [ $(uname) == "Darwin" ]; then + soname=$(cd local/scipy_openblas64/lib; ls libscipy_openblas*.dylib) + echo otool -D local/scipy_openblas64/lib/$soname + otool -D local/scipy_openblas64/lib/$soname + # issue 153: there is a ".0" in the install_name. Remove it + # also add a @rpath + install_name_tool -id @rpath/$soname local/scipy_openblas64/lib/$soname +fi + +rm -rf local/scipy_openblas64/lib/pkgconfig +echo "" >> LICENSE.txt +echo "----" >> LICENSE.txt +echo "" >> LICENSE.txt +if [ $(uname) == "Darwin" ]; then + cat tools/LICENSE_osx.txt >> LICENSE.txt +else + cat tools/LICENSE_linux.txt >> LICENSE.txt +fi + +if [ "${INTERFACE64}" != "1" ]; then + # rewrite the name of the project to scipy-openblas32 + # this is a hack, but apparently there is no other way to change the name + # of a pyproject.toml project + # + # use the BSD variant of sed -i and remove the backup + sed -e "s/openblas64/openblas32/" -i.bak pyproject.toml + rm *.bak + mv local/scipy_openblas64 local/scipy_openblas32 + sed -e "s/openblas_get_config64_/openblas_get_config/" -i.bak local/scipy_openblas32/__init__.py + sed -e "s/cflags =.*/cflags = '-DBLAS_SYMBOL_PREFIX=scipy_'/" -i.bak local/scipy_openblas32/__init__.py + sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__main__.py + sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__init__.py + rm local/scipy_openblas32/*.bak +fi + +rm -rf dist/* diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index dfd8bde5..cb2cf16a 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -2,5 +2,32 @@ # $INTERFACE64 ("1" or "0") # $PLAT (x86_64, i686, arm64, aarch64, s390x, ppc64le) +source tools/build_prepare.sh + $PYTHON -m pip wheel -w dist -v . +# The code below is for Travis use only. + +auditwheel repair -w dist --lib-sdir /lib dist/*.whl +rm dist/scipy_openblas*-none-any.whl +# Add an RPATH to libgfortran: +# https://github.com/pypa/auditwheel/issues/451 +if [ "$MB_ML_LIBC" == "musllinux" ]; then + apk add zip +else + yum install -y zip +fi +unzip dist/*.whl "*libgfortran*" +patchelf --force-rpath --set-rpath '$ORIGIN' */lib/libgfortran* +zip dist/*.whl */lib/libgfortran* + +# Test that the wheel works with a different python +PYTHON=python3.11 + +if [ "${INTERFACE64}" != "1" ]; then + $PYTHON -m pip install --no-index --find-links dist scipy_openblas32 + $PYTHON -m scipy_openblas32 +else + $PYTHON -m pip install --no-index --find-links dist scipy_openblas64 + $PYTHON -m scipy_openblas64 +fi \ No newline at end of file From ab8b10de730f4f09c2750f3d6fb5549bc7bb4499 Mon Sep 17 00:00:00 2001 From: ffgan Date: Mon, 20 Oct 2025 19:32:49 +0800 Subject: [PATCH 4/9] fix travis error Co-authored by: nijincheng@iscas.ac.cn; --- tools/build_steps.sh | 41 ++++++++++++++++++++++++++++++++++++-- tools/build_wheel.sh | 2 ++ tools/docker_build_wrap.sh | 4 +++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 5569eca1..8afaa653 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -288,12 +288,49 @@ function do_build_lib { $BUILD_PREFIX/lib/cmake/openblas } + +function build_lib_on_travis { + # OSX or manylinux build + # + # Input arg + # plat - one of i686, x86_64, arm64 + # interface64 - 1 if build with INTERFACE64 and SYMBOLSUFFIX + # nightly - 1 if building for nightlies + # + # Depends on globals + # BUILD_PREFIX - install suffix e.g. "/usr/local" + # MB_ML_VER + set -x + local plat=${1:-$PLAT} + local interface64=${2:-$INTERFACE64} + local nightly=${3:0} + local manylinux=${MB_ML_VER:-1} + + # Manylinux wrapper + local libc=${MB_ML_LIBC:-manylinux} + local docker_image=quay.io/pypa/${libc}${manylinux}_${plat} + docker pull $docker_image + # Docker sources this script, and runs `do_build_lib` + docker run --rm \ + -e BUILD_PREFIX="$BUILD_PREFIX" \ + -e PLAT="${plat}" \ + -e INTERFACE64="${interface64}" \ + -e NIGHTLY="${nightly}" \ + -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ + -e MB_ML_VER=${manylinux} \ + -e MB_ML_LIBC=${libc} \ + -v $PWD:/io \ + $docker_image /io/tools/docker_build_wrap.sh +} + + + function build_on_travis { if [ ${TRAVIS_EVENT_TYPE} == "cron" ]; then - build_lib "$PLAT" "$INTERFACE64" 1 + build_lib_on_travis "$PLAT" "$INTERFACE64" 1 version=$(cd OpenBLAS && git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml else - build_lib "$PLAT" "$INTERFACE64" 0 + build_lib_on_travis "$PLAT" "$INTERFACE64" 0 fi } diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index cb2cf16a..8eeafcab 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -2,6 +2,8 @@ # $INTERFACE64 ("1" or "0") # $PLAT (x86_64, i686, arm64, aarch64, s390x, ppc64le) +set -xe + source tools/build_prepare.sh $PYTHON -m pip wheel -w dist -v . diff --git a/tools/docker_build_wrap.sh b/tools/docker_build_wrap.sh index a385950c..20c68cd0 100755 --- a/tools/docker_build_wrap.sh +++ b/tools/docker_build_wrap.sh @@ -5,6 +5,8 @@ set -e # Change into root directory of repo -# cd /io +if [[ ! -e tools/build_steps.sh ]];then + cd /io +fi source tools/build_steps.sh do_build_lib "$PLAT" "" "$INTERFACE64" "$NIGHTLY" From 4a7066281e2f51144bff1e8bd7fd77c11aafa0ba Mon Sep 17 00:00:00 2001 From: ffgan Date: Mon, 20 Oct 2025 19:50:50 +0800 Subject: [PATCH 5/9] fix travis error Co-authored by: nijincheng@iscas.ac.cn; --- tools/build_wheel.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index 8eeafcab..e5a0f555 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -2,14 +2,18 @@ # $INTERFACE64 ("1" or "0") # $PLAT (x86_64, i686, arm64, aarch64, s390x, ppc64le) +# The code below is for Travis use only. + set -xe +if [[ ! -e tools/build_prepare.sh ]];then + cd /openblas +fi + source tools/build_prepare.sh $PYTHON -m pip wheel -w dist -v . -# The code below is for Travis use only. - auditwheel repair -w dist --lib-sdir /lib dist/*.whl rm dist/scipy_openblas*-none-any.whl # Add an RPATH to libgfortran: From fbb83f35d526f81f2e0217f54229391f54cb2194 Mon Sep 17 00:00:00 2001 From: ffgan Date: Mon, 20 Oct 2025 20:43:43 +0800 Subject: [PATCH 6/9] fix travis error Co-authored by: nijincheng@iscas.ac.cn; --- tools/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index e5a0f555..16cb9caa 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -15,7 +15,7 @@ source tools/build_prepare.sh $PYTHON -m pip wheel -w dist -v . auditwheel repair -w dist --lib-sdir /lib dist/*.whl -rm dist/scipy_openblas*-none-any.whl + # Add an RPATH to libgfortran: # https://github.com/pypa/auditwheel/issues/451 if [ "$MB_ML_LIBC" == "musllinux" ]; then From 01b3d6d7c0589ff8ea4e173c2560c6fc776e6281 Mon Sep 17 00:00:00 2001 From: ffgan Date: Mon, 20 Oct 2025 21:58:00 +0800 Subject: [PATCH 7/9] try to debug travis Co-authored by: nijincheng@iscas.ac.cn; --- tools/build_prepare.sh | 3 +++ tools/build_wheel.sh | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/build_prepare.sh b/tools/build_prepare.sh index 0d241c38..cb99d44c 100644 --- a/tools/build_prepare.sh +++ b/tools/build_prepare.sh @@ -67,3 +67,6 @@ if [ "${INTERFACE64}" != "1" ]; then fi rm -rf dist/* + + +echo "The build preparation is done." \ No newline at end of file diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index 16cb9caa..71e36bef 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -1,3 +1,4 @@ +#! /bin/bash # Needs: # $INTERFACE64 ("1" or "0") # $PLAT (x86_64, i686, arm64, aarch64, s390x, ppc64le) @@ -14,7 +15,11 @@ source tools/build_prepare.sh $PYTHON -m pip wheel -w dist -v . +echo "Repairing wheel with auditwheel" auditwheel repair -w dist --lib-sdir /lib dist/*.whl +echo "Wheel repaired." + +ls -l dist/ # Add an RPATH to libgfortran: # https://github.com/pypa/auditwheel/issues/451 @@ -27,6 +32,12 @@ unzip dist/*.whl "*libgfortran*" patchelf --force-rpath --set-rpath '$ORIGIN' */lib/libgfortran* zip dist/*.whl */lib/libgfortran* +echo "Final wheel contents:" + +ls -l dist/ + +echo "Testing the wheel" + # Test that the wheel works with a different python PYTHON=python3.11 @@ -36,4 +47,6 @@ if [ "${INTERFACE64}" != "1" ]; then else $PYTHON -m pip install --no-index --find-links dist scipy_openblas64 $PYTHON -m scipy_openblas64 -fi \ No newline at end of file +fi + +echo "Wheel test successful." \ No newline at end of file From 6b406e9672b99e0d9901b7b5df08994b95127da0 Mon Sep 17 00:00:00 2001 From: ffgan Date: Mon, 20 Oct 2025 22:09:47 +0800 Subject: [PATCH 8/9] final fix travis error Co-authored by: nijincheng@iscas.ac.cn; --- tools/build_wheel.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index 71e36bef..394dcb68 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -19,6 +19,8 @@ echo "Repairing wheel with auditwheel" auditwheel repair -w dist --lib-sdir /lib dist/*.whl echo "Wheel repaired." +rm dist/*none-linux*.whl + ls -l dist/ # Add an RPATH to libgfortran: From a11dabdb7a829e72bc2eb1bd11dc1208914cc35a Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 21 Oct 2025 15:26:27 +0300 Subject: [PATCH 9/9] some cleanup --- ci-before-build.sh | 27 +-------------------------- tools/build_steps.sh | 35 ++++------------------------------- 2 files changed, 5 insertions(+), 57 deletions(-) diff --git a/ci-before-build.sh b/ci-before-build.sh index a7f4d81e..a4a15941 100755 --- a/ci-before-build.sh +++ b/ci-before-build.sh @@ -69,15 +69,6 @@ set -e MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") DOWNLOADS_SDIR=downloads PYPY_URL=https://downloads.python.org/pypy -# For back-compatibility. We use the "ensurepip" module now -# instead of get-pip.py -GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py - -# Unicode width, default 32. Used here and in travis_linux_steps.sh -# In docker_build_wrap.sh it is passed in when calling "docker run" -# The docker test images also use it when choosing the python to run -# with, so it is passed in when calling "docker run" for tests. -UNICODE_WIDTH=${UNICODE_WIDTH:-32} if [ $(uname) == "Darwin" ]; then IS_MACOS=1; IS_OSX=1; @@ -106,23 +97,7 @@ unset -f cd unset -f pushd unset -f popd -function cmd_notexit { - # wraps a command, capturing its return code and preventing it - # from exiting the shell. Handles -e / +e modes. - # Parameters - # cmd - command - # any further parameters are passed to the wrapped command - # If called without an argument, it will exit the shell with an error - local cmd=$1 - if [ -z "$cmd" ];then echo "no command"; exit 1; fi - if [[ $- = *e* ]]; then errexit_set=true; fi - set +e - ("${@:1}") ; retval=$? - [[ -n $errexit_set ]] && set -e - return $retval -} - # Build OpenBLAS source build-openblas.sh -source tools/build_prepare.sh \ No newline at end of file +source tools/build_prepare.sh diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 8afaa653..1262e1d8 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -39,12 +39,7 @@ function before_build { # get_macpython_environment ${MB_PYTHON_VERSION} venv python3.9 -m venv venv source venv/bin/activate - # Since install_fortran uses `uname -a` to determine arch, - # force the architecture - arch -${PLAT} bash -s << EOF -source tools/gfortran_utils.sh -install_gfortran -EOF + alias gfortran=gfortran-15 # Deployment target set by gfortran_utils echo "Deployment target $MACOSX_DEPLOYMENT_TARGET" @@ -60,12 +55,6 @@ function clean_code_local { local build_commit=${2:-$BUILD_COMMIT} [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 - # The package $repo_dir may be a submodule. git submodules do not - # have a .git directory. If $repo_dir is copied around, tools like - # Versioneer which require that it be a git repository are unable - # to determine the version. Give submodule proper git directory - # XXX no need to do this - # fill_submodule "$repo_dir" pushd $repo_dir echo in $repo_dir git fetch origin --tags @@ -124,28 +113,12 @@ function build_lib { local interface64=${2:-$INTERFACE64} local nightly=${3:0} local manylinux=${MB_ML_VER:-1} - # Make directory to store built archive if [ -n "$IS_OSX" ]; then # Do build, add gfortran hash to end of name do_build_lib "$plat" "gf_${GFORTRAN_SHA:0:7}" "$interface64" "$nightly" - return + else + do_build_lib "$plat" "" "$interface64" "$nightly" fi - # Manylinux wrapper - local libc=${MB_ML_LIBC:-manylinux} - local docker_image=quay.io/pypa/${libc}${manylinux}_${plat} - # docker pull $docker_image - # # Docker sources this script, and runs `do_build_lib` - # docker run --rm \ - # -e BUILD_PREFIX="$BUILD_PREFIX" \ - # -e PLAT="${plat}" \ - # -e INTERFACE64="${interface64}" \ - # -e NIGHTLY="${nightly}" \ - # -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ - # -e MB_ML_VER=${manylinux} \ - # -e MB_ML_LIBC=${libc} \ - # -v $PWD:/io \ - # $docker_image /io/tools/docker_build_wrap.sh - bash ./tools/docker_build_wrap.sh } function patch_source { @@ -310,7 +283,7 @@ function build_lib_on_travis { local libc=${MB_ML_LIBC:-manylinux} local docker_image=quay.io/pypa/${libc}${manylinux}_${plat} docker pull $docker_image - # Docker sources this script, and runs `do_build_lib` + # run `do_build_lib` in the docker image docker run --rm \ -e BUILD_PREFIX="$BUILD_PREFIX" \ -e PLAT="${plat}" \