Merge pull request scipy#18012 from andyfaff/blas

BLD: bump OpenBLAS to v0.3.20-571-g3dec11c6 [wheel build]

Author: andyfaff

.github/workflows/windows.yml

@@ -46,45 +46,32 @@ jobs:
           gfortran --version
       - name: pip-packages
         run: |
-          pip install numpy==1.22.2 cython pybind11 pythran meson ninja pytest pytest-xdist pytest-timeout pooch
+          pip install numpy==1.22.2 cython pybind11 pythran meson ninja pytest pytest-xdist pytest-timeout pooch rich_click click doit pydevtool
       - name: openblas-libs
         run: |
           # Download and install pre-built OpenBLAS library
           # Built with mingw-w64, -ucrt -static.
           # https://github.com/matthew-brett/openblas-libs/blob/ucrt-build/build_openblas.ps1
-          choco install unzip -y
-          choco install wget -y
-          wget https://github.com/scipy/scipy-ci-artifacts/raw/main/openblas_32_if.zip
-          unzip -d c:\ openblas_32_if.zip
-          echo "PKG_CONFIG_PATH=c:\opt\openblas\if_32\64\lib\pkgconfig;" >> $env:GITHUB_ENV
-      - name: meson-configure
-        run: |
-          meson setup build --prefix=$PWD\build
-      - name: meson-build
-        run: |
-          ninja -j 2 -C build
-      - name: meson-install
+          choco install wget unzip -y --no-progress
+          wget -q https://anaconda.org/multibuild-wheels-staging/openblas-libs/v0.3.20-571-g3dec11c6/download/openblas-v0.3.20-571-g3dec11c6-win_amd64-gcc_10_3_0.zip
+          unzip -d c:\opt openblas-v0.3.20-571-g3dec11c6-win_amd64-gcc_10_3_0.zip
+          echo "PKG_CONFIG_PATH=c:\opt\64\lib\pkgconfig;" >> $env:GITHUB_ENV
+      - name: build
         run: |
+          echo "SCIPY_USE_PROPACK=1" >> $env:GITHUB_ENV
+          python dev.py build -j 2 --win-cp-openblas
+
+          # following steps are required because the build step does not put
+          # the *.pyd files in build-install! Furthermore the --win-cp-openblas
+          # option does not seem to copy libopenblas*.dll, so manually copy it.
           cd build
           meson install
-      - name: build-path
-        run: |
-          echo "installed_path=$PWD\build\Lib\site-packages" >> $env:GITHUB_ENV
-      - name: post-install
-        run: |
-          $scipy_path = "${env:installed_path}\scipy"
-          $libs_path = "${scipy_path}\.libs"
-          mkdir ${libs_path}
-          $ob_path = (pkg-config --variable libdir openblas) -replace "lib", "bin"
-          cp $ob_path/*.dll $libs_path
-          # Write _distributor_init.py to scipy dir to load .libs DLLs.
-          & python tools\openblas_support.py --write-init $scipy_path
-      - name: prep-test
-        run: |
-          echo "PYTHONPATH=${env:installed_path}" >> $env:GITHUB_ENV
-          echo "SCIPY_USE_PROPACK=1" >> $env:GITHUB_ENV
+          cd ..
+          cp C:\opt\64\bin\*.dll $pwd\build-install\Lib\site-packages\scipy\.libs\
+          echo "PYTHONPATH=$PWD\build-install\Lib\site-packages" >> $env:GITHUB_ENV
+          python tools\openblas_support.py --write-init $PWD\build-install\Lib\site-packages\scipy\
       - name: test
         run: |
-          mkdir tmp
-          cd tmp
-          python -c 'import scipy; scipy.test()'
+          Get-ChildItem env:
+          cd $env:PYTHONPATH
+          python -c "import sys; import scipy; sys.exit(not scipy.test())"

azure-pipelines.yml

@@ -21,7 +21,7 @@ pr:
 # the version of OpenBLAS used is currently 0.3.21
 # and should be updated to match scipy-wheels as appropriate
 variables:
-    openblas_version: 0.3.21
+    openblas_version: 0.3.21.dev
     pre_wheels: https://pypi.anaconda.org/scipy-wheels-nightly/simple
     CCACHE_DIR: $(Pipeline.Workspace)/ccache
     SCIPY_AVAILABLE_MEM: 3G

dev.py

@@ -143,9 +143,12 @@ def task_meta(cls, **kwargs):
     "cmd": "italic gray50",
 })
 
-
-class EMOJI:
-    cmd = ":computer:"
+if sys.platform == 'win32':
+    class EMOJI:
+        cmd = ">"
+else:
+    class EMOJI:
+        cmd = ":computer:"
 
 
 rich_click.STYLE_ERRORS_SUGGESTION = "yellow italic"

pyproject.toml

@@ -165,4 +165,5 @@ select = "*-win_amd64"
 # An alternative is to set CMAKE_PREFIX_PATH="c:/opt/openblas/if_32/32"
 # Don't use double backslash for path separators, they don't get passed
 # to the build correctly
+# environment = { CMAKE_PREFIX_PATH="c:/opt/64" }
 environment = { PKG_CONFIG_PATH="c:/opt/64/lib/pkgconfig" }

scipy/linalg/tests/test_decomp.py

@@ -46,7 +46,10 @@
 
 from scipy._lib._testutils import check_free_memory
 from scipy.linalg.blas import HAS_ILP64
-
+try:
+    from scipy.__config__ import CONFIG
+except ImportError:
+    CONFIG = None
 
 def _random_hermitian_matrix(n, posdef=False, dtype=float):
     "Generate random sym/hermitian array of the given size n"
@@ -2043,12 +2046,22 @@ def test_2x2(self):
         assert_array_almost_equal(h2, b)
 
 
+blas_provider = blas_version = None
+if CONFIG is not None:
+    blas_provider = CONFIG['Build Dependencies']['blas']['name']
+    blas_version = CONFIG['Build Dependencies']['blas']['version']
+
+
 class TestQZ:
     def setup_method(self):
         seed(12345)
 
-    @pytest.mark.xfail(sys.platform == 'darwin',
-        reason="gges[float32] broken for OpenBLAS on macOS, see gh-16949")
+    @pytest.mark.xfail(
+        sys.platform == 'darwin' and
+        blas_provider == 'openblas' and
+        blas_version < "0.3.21.dev",
+        reason="gges[float32] broken for OpenBLAS on macOS, see gh-16949"
+    )
     def test_qz_single(self):
         n = 5
         A = random([n, n]).astype(float32)

scipy/linalg/tests/test_lapack.py

@@ -23,8 +23,11 @@
 from scipy.linalg.lapack import _compute_lwork
 from scipy.stats import ortho_group, unitary_group
 
-
 import scipy.sparse as sps
+try:
+    from scipy.__config__ import CONFIG
+except ImportError:
+    CONFIG = None
 
 try:
     from scipy.linalg import _clapack as clapack
@@ -37,6 +40,11 @@
 COMPLEX_DTYPES = [np.complex64, np.complex128]
 DTYPES = REAL_DTYPES + COMPLEX_DTYPES
 
+blas_provider = blas_version = None
+if CONFIG is not None:
+    blas_provider = CONFIG['Build Dependencies']['blas']['name']
+    blas_version = CONFIG['Build Dependencies']['blas']['version']
+
 
 def generate_random_dtype_array(shape, dtype):
     # generates a random matrix of desired data type of shape
@@ -3052,7 +3060,12 @@ def test_trexc_NAG(t, ifst, ilst, expect):
 
 @pytest.mark.parametrize('dtype', DTYPES)
 def test_gges_tgexc(dtype):
-    if dtype == np.float32 and sys.platform == 'darwin':
+    if (
+        dtype == np.float32 and
+        sys.platform == 'darwin' and
+        blas_provider == 'openblas' and
+        blas_version < '0.3.21.dev'
+    ):
         pytest.xfail("gges[float32] broken for OpenBLAS on macOS, see gh-16949")
 
     seed(1234)
@@ -3224,7 +3237,12 @@ def test_trsen_NAG(t, q, select, expect, expect_s, expect_sep):
 
 @pytest.mark.parametrize('dtype', DTYPES)
 def test_gges_tgsen(dtype):
-    if dtype == np.float32 and sys.platform == 'darwin':
+    if (
+        dtype == np.float32 and
+        sys.platform == 'darwin' and
+        blas_provider == 'openblas' and
+        blas_version < '0.3.21.dev'
+    ):
         pytest.xfail("gges[float32] broken for OpenBLAS on macOS, see gh-16949")
 
     seed(1234)

scipy/linalg/tests/test_solvers.py

@@ -510,8 +510,11 @@ def test_solve_discrete_are():
     #
     min_decimal = (12, 14, 13, 14, 13, 16, 18, 14, 14, 13,
                    14, 13, 13, 14, 12, 2, 5, 6, 10)
+    max_tol = [1.5 * 10**-ind for ind in min_decimal]
+    # relaxed tolerance in gh-18012 after bump to OpenBLAS
+    max_tol[11] = 2.5e-13
 
-    def _test_factory(case, dec):
+    def _test_factory(case, atol):
         """Checks if X = A'XA-(A'XB)(R+B'XB)^-1(B'XA)+Q) is true"""
         a, b, q, r, knownfailure = case
         if knownfailure:
@@ -522,10 +525,14 @@ def _test_factory(case, dec):
         res -= a.conj().T.dot(x.dot(b)).dot(
                     solve(r+b.conj().T.dot(x.dot(b)), b.conj().T).dot(x.dot(a))
                     )
-        assert_array_almost_equal(res, np.zeros_like(res), decimal=dec)
+        # changed from
+        # assert_array_almost_equal(res, np.zeros_like(res), decimal=dec)
+        # in gh-18012 as it's easier to relax a tolerance and allclose is
+        # preferred
+        assert_allclose(res, np.zeros_like(res), atol=atol)
 
     for ind, case in enumerate(cases):
-        _test_factory(case, min_decimal[ind])
+        _test_factory(case, max_tol[ind])
 
     # An infeasible example taken from https://arxiv.org/abs/1505.04861v1
     A = np.triu(np.ones((3, 3)))

tools/openblas_support.py

@@ -13,8 +13,8 @@
 from urllib.request import urlopen, Request
 from urllib.error import HTTPError
 
-OPENBLAS_V = '0.3.21'
-OPENBLAS_LONG = 'v0.3.21'
+OPENBLAS_V = '0.3.21.dev'
+OPENBLAS_LONG = 'v0.3.20-571-g3dec11c6'
 BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs'
 BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download'
 SUPPORTED_PLATFORMS = [