diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 067eba1..f56d938 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -4,14 +4,15 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: + - main + - maintenance/** workflow_dispatch: null schedule: - cron: '0 0 * * 0' env: OPENBLAS_COMMIT: "v0.3.30-349-gf6df9beb" - MACOSX_DEPLOYMENT_TARGET: 10.9 jobs: build: diff --git a/.github/workflows/windows-arm.yml b/.github/workflows/windows-arm.yml index 2b56e82..bdbf044 100644 --- a/.github/workflows/windows-arm.yml +++ b/.github/workflows/windows-arm.yml @@ -4,7 +4,9 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: + - main + - maintenance/** env: OPENBLAS_COMMIT: "v0.3.30-349-gf6df9beb" diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 09599c6..3687222 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -4,7 +4,9 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: + - main + - maintenance/** workflow_dispatch: null env: diff --git a/ci-before-build.sh b/ci-before-build.sh index f2dc8a3..8df50e1 100755 --- a/ci-before-build.sh +++ b/ci-before-build.sh @@ -4,8 +4,6 @@ # Most of the content in this file comes from https://github.com/multi-build/multibuild, with some modifications # Follow the license below - - # .. _license: # ********************* @@ -41,9 +39,6 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -#! /bin/bash set -xe if [[ "$NIGHTLY" = "true" ]]; then @@ -55,40 +50,6 @@ if [[ "$NIGHTLY" = "true" ]]; then sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml fi - -#!/bin/bash -# Utilities for both OSX and Docker Linux -# python or python3 should be on the PATH - -# Only source common_utils once -if [ -n "$COMMON_UTILS_SOURCED" ]; then - return -fi -COMMON_UTILS_SOURCED=1 - -# Turn on exit-if-error -set -e - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -DOWNLOADS_SDIR=downloads -PYPY_URL=https://downloads.python.org/pypy - -if [ $(uname) == "Darwin" ]; then - IS_MACOS=1; IS_OSX=1; -else - # In the manylinux_2_24 image, based on Debian9, "python" is not installed - # so link in something for the various system calls before PYTHON_EXE is set - which python || export PATH=/opt/python/cp39-cp39/bin:$PATH - - if [ "$MB_ML_LIBC" == "musllinux" ]; then - IS_ALPINE=1; - MB_ML_VER=${MB_ML_VER:-"_1_2"} - else - # Default Manylinux version - MB_ML_VER=${MB_ML_VER:-2014} - fi -fi - # Work round bug in travis xcode image described at # https://github.com/direnv/direnv/issues/210 shell_session_update() { :; } @@ -103,4 +64,5 @@ unset -f popd # Build OpenBLAS source build-openblas.sh +# Build wheel source tools/build_prepare.sh diff --git a/pyproject.toml b/pyproject.toml index c956d4c..0efd0ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "scipy-openblas64" # v0.3.30-349-gf6df9beb -version = "0.3.30.349.0" +version = "0.3.30.349.1" requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" @@ -46,7 +46,6 @@ repair-wheel-command = "bash ci-repair-wheel.sh {dest_dir} {wheel}" test-command = "cd {package} && bash ci-test.sh " environment-pass = [ "OPENBLAS_COMMIT", - "MACOSX_DEPLOYMENT_TARGET", "NIGHTLY", "MB_ML_LIBC", "MB_ML_VER", diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 6aacb01..81743c2 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -1,3 +1,5 @@ +#!bash + # Build script for manylinux and OSX BUILD_PREFIX=${BUILD_PREFIX:-/usr/local} @@ -5,27 +7,9 @@ ROOT_DIR=$(dirname $(dirname "${BASH_SOURCE[0]}")) MB_PYTHON_VERSION=3.9 -function any_python { - for cmd in $PYTHON_EXE python3 python; do - if [ -n "$(type -t $cmd)" ]; then - echo $cmd - return - fi - done - echo "Could not find python or python3" - exit 1 -} - -function get_os { - # Report OS as given by uname - # Use any Python that comes to hand. - $(any_python) -c 'import platform; print(platform.uname()[0])' -} - - function before_build { # Manylinux Python version set in build_lib - if [ -n "$IS_OSX" ]; then + if [ "$(uname -s)" == "Darwin" ]; then if [ ! -e /usr/local/lib ]; then sudo mkdir -p /usr/local/lib sudo chmod 777 /usr/local/lib @@ -39,29 +23,39 @@ function before_build { # get_macpython_environment ${MB_PYTHON_VERSION} venv python3.9 -m venv venv source venv/bin/activate - alias gfortran=gfortran-15 + + unalias gfortran 2>/dev/null || true + source tools/gfortran_utils.sh + download_and_unpack_gfortran ${PLAT} native + export FC=/opt/gfortran/gfortran-darwin-${PLAT}-native/bin/gfortran + which ${FC} + ${FC} --version + local libdir=/opt/gfortran/gfortran-darwin-${PLAT}-native/lib + # Remove conflicting shared objects + rm -fv ${libdir}/libiconv* + export FFLAGS="-L${libdir} -Wl,-rpath,${libdir}" + # Not clear why this is needed for tests on arm64... + export LDFLAGS="$FFLAGS" + # Deployment target set by gfortran_utils echo "Deployment target $MACOSX_DEPLOYMENT_TARGET" # Build the objconv tool - (cd ${ROOT_DIR}/objconv && bash ../tools/build_objconv.sh) + if [[ ! -x objconv/objconv ]]; then + (cd ${ROOT_DIR}/objconv && bash ../tools/build_objconv.sh) + fi fi } function clean_code { set -ex - # Copied from common_utils.sh, with added debugging local build_commit=$1 [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 pushd OpenBLAS git fetch origin --tags - echo after git fetch origin git checkout $build_commit - echo after git checkout $build_commit git clean -fxd - echo after git clean git submodule update --init --recursive - echo after git submodule update popd } @@ -108,12 +102,7 @@ function build_lib { local interface64=${2:-$INTERFACE64} local nightly=${3:0} local manylinux=${MB_ML_VER:-1} - if [ -n "$IS_OSX" ]; then - # Do build, add gfortran hash to end of name - do_build_lib "$plat" "gf_${GFORTRAN_SHA:0:7}" "$interface64" "$nightly" - else - do_build_lib "$plat" "" "$interface64" "$nightly" - fi + do_build_lib "$plat" "$interface64" "$nightly" } function patch_source { @@ -129,8 +118,6 @@ function do_build_lib { # Build openblas lib # Input arg # plat - one of i686, x86_64, arm64 - # suffix (optional) - suffix for output archive name - # Suffix added with hyphen prefix # interface64 (optional) - whether to build ILP64 openblas # with 64_ symbol suffix # nightly (optional) - whether to build for nightlies @@ -138,10 +125,9 @@ function do_build_lib { # Depends on globals # BUILD_PREFIX - install suffix e.g. "/usr/local" local plat=$1 - local suffix=$2 - local interface64=$3 - local nightly=$4 - case $(get_os)-$plat in + local interface64=$2 + local nightly=$3 + case $(uname -s)-$plat in Linux-x86_64) local bitness=64 local target="PRESCOTT" @@ -150,16 +136,8 @@ function do_build_lib { Darwin-x86_64) local bitness=64 local target="CORE2" - # Use gfortran-11 - unalias gfortran - # Since install_fortran uses `uname -a` to determine arch, - # force the architecture - arch -${PLAT} bash -s << EOF -source ${ROOT_DIR}/gfortran-install/gfortran_utils.sh -install_gfortran -EOF - export DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH CFLAGS="$CFLAGS -arch x86_64" + MACOSX_DEPLOYMENT_TARGET="10.9" export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} local dynamic_list="CORE2 NEHALEM SANDYBRIDGE HASWELL SKYLAKEX" ;; @@ -171,8 +149,8 @@ EOF Linux-aarch64) local bitness=64 local target="ARMV8" - # manylinux2014 image uses gcc-10, which miscompiles ARMV8SVE and up if [ "$MB_ML_VER" == "2014" ]; then + # manylinux2014 image uses gcc-10, which miscompiles ARMV8SVE and up echo setting DYNAMIC_LIST for manylinux2014 to ARMV8 only local dynamic_list="ARMV8" fi @@ -183,7 +161,6 @@ EOF CFLAGS="$CFLAGS -ftrapping-math -mmacos-version-min=11.0" MACOSX_DEPLOYMENT_TARGET="11.0" export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} - export DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH ;; *-s390x) local bitness=64 @@ -201,9 +178,6 @@ EOF 1) local interface_flags="INTERFACE64=1 SYMBOLSUFFIX=64_ LIBNAMESUFFIX=64_ OBJCONV=$PWD/objconv/objconv"; local symbolsuffix="64_"; - if [ -n "$IS_OSX" ]; then - $PWD/objconv/objconv --help - fi ;; *) local interface_flags="OBJCONV=$PWD/objconv/objconv" @@ -246,8 +220,6 @@ EOF fi mv $BUILD_PREFIX/lib/pkgconfig/openblas*.pc $BUILD_PREFIX/lib/pkgconfig/scipy-openblas.pc local plat_tag=$(get_plat_tag $plat) - local suff="" - [ -n "$suffix" ] && suff="-$suffix" if [ "$interface64" = "1" ]; then # OpenBLAS does not install the symbol suffixed static library, # do it ourselves @@ -260,7 +232,7 @@ EOF rm $BUILD_PREFIX/lib/pkgconfig/scipy-openblas.pc.bak fi - local out_name="openblas${symbolsuffix}-${version}-${plat_tag}${suff}.tar.gz" + local out_name="openblas${symbolsuffix}-${version}-${plat_tag}.tar.gz" tar zcvf libs/$out_name \ $BUILD_PREFIX/include/*blas* \ $BUILD_PREFIX/include/*lapack* \ diff --git a/tools/docker_build_wrap.sh b/tools/docker_build_wrap.sh index 20c68cd..4522340 100755 --- a/tools/docker_build_wrap.sh +++ b/tools/docker_build_wrap.sh @@ -1,7 +1,8 @@ #!/bin/bash # Depends on: -# BUILD_PREFIX # PLAT +# INTERFACE64 (could be missing or empty) +# NIGHTLY (could be missing or empty) set -e # Change into root directory of repo @@ -9,4 +10,4 @@ if [[ ! -e tools/build_steps.sh ]];then cd /io fi source tools/build_steps.sh -do_build_lib "$PLAT" "" "$INTERFACE64" "$NIGHTLY" +do_build_lib "$PLAT" "$INTERFACE64" "$NIGHTLY" diff --git a/tools/gfortran_utils.sh b/tools/gfortran_utils.sh index b7127f5..30040a6 100644 --- a/tools/gfortran_utils.sh +++ b/tools/gfortran_utils.sh @@ -1,3 +1,4 @@ +#!bash # this file come from https://github.com/MacPython/gfortran-install # Follow the license below @@ -16,139 +17,51 @@ # Bash utilities for use with gfortran -ARCHIVE_SDIR="${ARCHIVE_SDIR:-archives}" - -GF_UTIL_DIR=$(dirname "${BASH_SOURCE[0]}") - -function get_distutils_platform { - # Report platform as in form of distutils get_platform. - # This is like the platform tag that pip will use. - # Modify fat architecture tags on macOS to reflect compiled architecture - - # Deprecate this function once get_distutils_platform_ex is used in all - # downstream projects - local plat=$1 - case $plat in - i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; - *) echo Did not recognize plat $plat; return 1 ;; - esac - local uname=${2:-$(uname)} - if [ "$uname" != "Darwin" ]; then - if [ "$plat" == "intel" ]; then - echo plat=intel not allowed for Manylinux - return 1 - fi - echo "manylinux1_$plat" - return - fi - # macOS 32-bit arch is i386 - [ "$plat" == "i686" ] && plat="i386" - local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) - echo "macosx_${target}_${plat}" -} - -function get_distutils_platform_ex { - # Report platform as in form of distutils get_platform. - # This is like the platform tag that pip will use. - # Modify fat architecture tags on macOS to reflect compiled architecture - # For non-darwin, report manylinux version - local plat=$1 - local mb_ml_ver=${MB_ML_VER:-1} - case $plat in - i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; - *) echo Did not recognize plat $plat; return 1 ;; - esac - local uname=${2:-$(uname)} - if [ "$uname" != "Darwin" ]; then - if [ "$plat" == "intel" ]; then - echo plat=intel not allowed for Manylinux - return 1 - fi - echo "manylinux${mb_ml_ver}_${plat}" - return - fi - # macOS 32-bit arch is i386 - [ "$plat" == "i686" ] && plat="i386" - local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) - echo "macosx_${target}_${plat}" -} - -function get_macosx_target { - # Report MACOSX_DEPLOYMENT_TARGET as given by distutils get_platform. - python3 -c "import sysconfig as s; print(s.get_config_vars()['MACOSX_DEPLOYMENT_TARGET'])" -} - -function check_gfortran { - # Check that gfortran exists on the path - if [ -z "$(which gfortran)" ]; then - echo Missing gfortran - exit 1 - fi -} - -function get_gf_lib_for_suf { - local suffix=$1 - local prefix=$2 - local plat=${3:-$PLAT} - local uname=${4:-$(uname)} - if [ -z "$prefix" ]; then echo Prefix not defined; exit 1; fi - local plat_tag=$(get_distutils_platform_ex $plat $uname) - if [ -n "$suffix" ]; then suffix="-$suffix"; fi - local fname="$prefix-${plat_tag}${suffix}.tar.gz" - local out_fname="${ARCHIVE_SDIR}/$fname" - [ -s $out_fname ] || (echo "$out_fname is empty"; exit 24) - echo "$out_fname" -} - if [ "$(uname)" == "Darwin" ]; then - mac_target=${MACOSX_DEPLOYMENT_TARGET:-$(get_macosx_target)} - export MACOSX_DEPLOYMENT_TARGET=$mac_target - # Keep this for now as some builds might depend on this being - # available before install_gfortran is called - export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e # Set SDKROOT env variable if not set export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} function download_and_unpack_gfortran { - local arch=$1 - local type=$2 - curl -L -O https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-${arch}-${type}.tar.gz - case ${arch}-${type} in - arm64-native) - export GFORTRAN_SHA=0d5c118e5966d0fb9e7ddb49321f63cac1397ce8 - ;; - arm64-cross) - export GFORTRAN_SHA=527232845abc5af21f21ceacc46fb19c190fe804 - ;; - x86_64-native) - export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e - ;; - x86_64-cross) - export GFORTRAN_SHA=107604e57db97a0ae3e7ca7f5dd722959752f0b3 - ;; - esac + local arch=$1 + local type=$2 + case ${arch}-${type} in + arm64-native) + export GFORTRAN_SHA=0d5c118e5966d0fb9e7ddb49321f63cac1397ce8 + ;; + arm64-cross) + export GFORTRAN_SHA=527232845abc5af21f21ceacc46fb19c190fe804 + ;; + x86_64-native) + export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e + ;; + x86_64-cross) + export GFORTRAN_SHA=107604e57db97a0ae3e7ca7f5dd722959752f0b3 + ;; + *) + echo unknown ${arch}-${type} + exit 1 + ;; + esac + if [[ ! -e gfortran-darwin-${arch}-${type}.tar.gz ]]; then + curl -L -O https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-${arch}-${type}.tar.gz + fi if [[ "$(shasum gfortran-darwin-${arch}-${type}.tar.gz)" != "${GFORTRAN_SHA} gfortran-darwin-${arch}-${type}.tar.gz" ]]; then echo "shasum mismatch for gfortran-darwin-${arch}-${type}" exit 1 fi - sudo mkdir -p /opt/ - sudo cp "gfortran-darwin-${arch}-${type}.tar.gz" /opt/gfortran-darwin-${arch}-${type}.tar.gz - pushd /opt - sudo tar -xvf gfortran-darwin-${arch}-${type}.tar.gz - sudo rm gfortran-darwin-${arch}-${type}.tar.gz + if [[ ! -d /opt/gfortran ]]; then + sudo mkdir -p /opt/gfortran + sudo chmod 777 /opt/gfortran + fi + cp "gfortran-darwin-${arch}-${type}.tar.gz" /opt/gfortran/gfortran-darwin-${arch}-${type}.tar.gz + pushd /opt/gfortran + tar -xvf gfortran-darwin-${arch}-${type}.tar.gz + rm gfortran-darwin-${arch}-${type}.tar.gz popd - if [[ "${type}" == "native" ]]; then - # Link these into /usr/local so that there's no need to add rpath or -L - for f in libgfortran.dylib libgfortran.5.dylib libgcc_s.1.dylib libgcc_s.1.1.dylib libquadmath.dylib libquadmath.0.dylib; do - ln -sf /opt/gfortran-darwin-${arch}-${type}/lib/$f /usr/local/lib/$f - done - # Add it to PATH - ln -sf /opt/gfortran-darwin-${arch}-${type}/bin/gfortran /usr/local/bin/gfortran - fi } function install_arm64_cross_gfortran { - download_and_unpack_gfortran arm64 cross + download_and_unpack_gfortran arm64 cross export FC_ARM64="$(find /opt/gfortran-darwin-arm64-cross/bin -name "*-gfortran")" local libgfortran="$(find /opt/gfortran-darwin-arm64-cross/lib -name libgfortran.dylib)" local libdir=$(dirname $libgfortran) @@ -158,26 +71,17 @@ if [ "$(uname)" == "Darwin" ]; then export FC=$FC_ARM64 fi } + function install_gfortran { download_and_unpack_gfortran $(uname -m) native - check_gfortran - if [[ "${PLAT:-}" == "universal2" || "${PLAT:-}" == "arm64" ]]; then + if [[ "${PLAT:-}" == "universal2" ]]; then install_arm64_cross_gfortran fi } - function get_gf_lib { - # Get lib with gfortran suffix - get_gf_lib_for_suf "gf_${GFORTRAN_SHA:0:7}" $@ - } else function install_gfortran { # No-op - already installed on manylinux image - check_gfortran - } - - function get_gf_lib { - # Get library with no suffix - get_gf_lib_for_suf "" $@ + : } fi diff --git a/tools/local_build.sh b/tools/local_build.sh index 4754c2c..441375d 100755 --- a/tools/local_build.sh +++ b/tools/local_build.sh @@ -1,3 +1,5 @@ +#!bash + # Replicate the workflow from posix.yml locally on posix # This may bitrot, compare it to the original file before using @@ -5,8 +7,8 @@ set -e # Set extra env if [[ $(uname) == "Darwin" ]]; then - # Force x86_64 - export PLAT=x86_64 + # export PLAT=x86_64 + export PLAT=arm64 elif [[ $(uname -m) == "x86_64" ]]; then echo got x86_64 export PLAT=x86_64 @@ -17,13 +19,8 @@ elif [[ $(uname -m) == arm64 ]]; then else echo got nothing exit -1 - export LDFLAGS="-L/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/lib" - export LIBRARY_PATH="-L/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/lib" - export PLAT=x86_64 - # export PLAT=arm64 - export SUFFIX=gf_c469a42 fi -export OPENBLAS_COMMIT="v0.3.30" +export OPENBLAS_COMMIT="v0.3.30-349-gf6df9beb" # export MB_ML_LIBC=musllinux # export MB_ML_VER=_1_2