Add nightly wheel builder (#6957)

.github/workflows/build_wheels.yml

@@ -0,0 +1,238 @@
+name: Build and Test Wheels
+on:
+  workflow_call:
+
+env:
+  CIBW_BUILD_VERBOSITY: 2
+  CIBW_TEST_REQUIRES: "-r requirements/default.txt -r requirements/test.txt"
+  CIBW_TEST_COMMAND: pytest --pyargs skimage
+  CIBW_ENVIRONMENT: PIP_PREFER_BINARY=1
+
+permissions:
+  contents: read
+
+jobs:
+  build_linux_38_and_above_wheels:
+    name: Build python ${{ matrix.cibw_python }} ${{ matrix.cibw_arch }} wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+        cibw_manylinux: [manylinux2014]
+        cibw_arch: ["x86_64"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: "3.9"
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel
+      - name: Build the wheel
+        run: |
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: ${{ matrix.cibw_python }}
+          CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }}
+          CIBW_SKIP: "*-musllinux_*"
+          CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.cibw_manylinux }}
+          CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.cibw_manylinux }}
+          CIBW_ENVIRONMENT_PASS_LINUX: SKIMAGE_LINK_FLAGS
+          SKIMAGE_LINK_FLAGS: "-Wl,--strip-debug"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: ./dist/*.whl
+
+  build_linux_aarch64_wheels:
+    name: Build python ${{ matrix.cibw_python }} aarch64 wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+        cibw_manylinux: [manylinux2014]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: "3.9"
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel
+      - name: Build the wheel
+        run: |
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: ${{ matrix.cibw_python }}
+          CIBW_SKIP: "*-musllinux_*"
+          # skip aarch64 tests which often exceed the 6 hour time limit
+          CIBW_TEST_SKIP: "*_aarch64"
+          CIBW_ARCHS_LINUX: aarch64
+          CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.cibw_manylinux }}
+          CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.cibw_manylinux }}
+          CIBW_ENVIRONMENT_PASS_LINUX: SKIMAGE_LINK_FLAGS
+          SKIMAGE_LINK_FLAGS: "-Wl,--strip-debug"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: ./dist/*.whl
+
+  build_macos_wheels:
+    name: Build python ${{ matrix.cibw_python }} ${{ matrix.cibw_arch }} wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest]
+        cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+        # TODO: add "universal2" once a universal2 libomp is available
+        cibw_arch: ["x86_64", "arm64"]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: "3.9"
+
+      # See:
+      # https://cibuildwheel.readthedocs.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64
+      # https://github.com/pypa/cibuildwheel/issues/1414
+      - name: Install experimental MacOSX Py38
+        if: startsWith(matrix.cibw_python, 'cp38') && (matrix.cibw_arch == 'arm64')
+        run: |
+          curl -o /tmp/Python38.pkg https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg
+          sudo installer -pkg /tmp/Python38.pkg -target /
+          sh "/Applications/Python 3.8/Install Certificates.command"
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel
+
+      - name: Build wheels for CPython Mac OS
+        run: |
+          # Make sure to use a libomp version binary compatible with the oldest
+          # supported version of the macos SDK as libomp will be vendored into
+          # the scikit-image wheels for macos. The list of binaries are in
+          # https://packages.macports.org/libomp/.  Currently, the oldest
+          # supported macos version is: High Sierra / 10.13. When upgrading
+          # this, be sure to update the MACOSX_DEPLOYMENT_TARGET environment
+          # variable accordingly. Note that Darwin_17 == High Sierra / 10.13.
+          #
+          # We need to set both MACOS_DEPLOYMENT_TARGET and MACOSX_DEPLOYMENT_TARGET
+          # until there is a new release with this commit:
+          # https://github.com/mesonbuild/meson-python/pull/309
+          if [[ "$CIBW_ARCHS_MACOS" == arm64 ]]; then
+              # SciPy requires 12.0 on arm to prevent kernel panics
+              # https://github.com/scipy/scipy/issues/14688
+              # so being conservative, we just do the same here
+              export MACOSX_DEPLOYMENT_TARGET=12.0
+              export MACOS_DEPLOYMENT_TARGET=12.0
+              OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-arm64/llvm-openmp-11.1.0-hf3c4609_1.tar.bz2"
+          else
+              export MACOSX_DEPLOYMENT_TARGET=10.9
+              export MACOS_DEPLOYMENT_TARGET=10.9
+              OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-64/llvm-openmp-11.1.0-hda6cdc1_1.tar.bz2"
+          fi
+          echo MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET}
+          echo MACOS_DEPLOYMENT_TARGET=${MACOS_DEPLOYMENT_TARGET}
+
+          # use conda to install llvm-openmp
+          # Note that we do NOT activate the conda environment, we just add the
+          # library install path to CFLAGS/CXXFLAGS/LDFLAGS below.
+          sudo conda create -n build $OPENMP_URL
+          PREFIX="/usr/local/miniconda/envs/build"
+          export CC=/usr/bin/clang
+          export CXX=/usr/bin/clang++
+          export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"
+          export CFLAGS="$CFLAGS -Wno-implicit-function-declaration -I$PREFIX/include"
+          export CXXFLAGS="$CXXFLAGS -I$PREFIX/include"
+          export LDFLAGS="$LDFLAGS -Wl,-S -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp"
+
+          python -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: ${{ matrix.cibw_python }}
+          CIBW_ARCHS_MACOS: ${{ matrix.cibw_arch }}
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
+          CIBW_MANYLINUX_I686_IMAGE: manylinux1
+          CIBW_TEST_SKIP: "*-macosx_arm64"
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: ./dist/*.whl
+
+  build_windows_wheels:
+    name: Build ${{ matrix.cibw_python }} ${{ matrix.cibw_arch }} wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest]
+        cibw_arch: ["AMD64"]
+        cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: "3.9"
+
+      - name: Install cibuildwheel and add clang-cl to path
+        run: |
+          python -m pip install cibuildwheel
+
+      - name: Build AMD64 Windows wheels for CPython
+        if: matrix.cibw_arch == 'AMD64'
+        # To avoid "LINK : fatal error LNK1158: cannot run 'rc.exe'"
+        # we explicitly add rc.exe to path using the method from:
+        # https://github.com/actions/virtual-environments/issues/294#issuecomment-588090582
+        # with additional -arch=x86 flag to vsdevcmd.bat
+        run: |
+          function Invoke-VSDevEnvironment {
+            $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
+              $installationPath = & $vswhere -prerelease -legacy -latest -property installationPath
+              $Command = Join-Path $installationPath "Common7\Tools\vsdevcmd.bat"
+            & "${env:COMSPEC}" /s /c "`"$Command`" -arch=amd64 -no_logo && set" | Foreach-Object {
+                  if ($_ -match '^([^=]+)=(.*)') {
+                      [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2])
+                  }
+              }
+          }
+          Invoke-VSDevEnvironment
+          Get-Command rc.exe | Format-Table -AutoSize
+          python -m cibuildwheel --output-dir dist
+        env:
+          # define CC, CXX so meson will use clang-cl instead of MSVC
+          CC: clang-cl
+          CXX: clang-cl
+          CIBW_BUILD: ${{ matrix.cibw_python }}
+          CIBW_ARCHS_WINDOWS: ${{ matrix.cibw_arch }}
+          # -Wl,-S equivalent to gcc's -Wl,--strip-debug
+          LDFLAGS: "-Wl,-S"
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: ./dist/*.whl

.github/workflows/cron_weekly.yml

@@ -0,0 +1,68 @@
+# Workflow to build and test against nightly wheels
+name: Test Nightly
+
+on:
+  [push]
+  #  schedule:
+  #    # run every Monday at 4am UTC
+  #    - cron: "0 4 * * 1"
+
+permissions:
+  contents: read
+
+jobs:
+  linux-${{ matrix.python-version }}:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build package
+        env:
+          # Set variables here that are used by the build script below
+          # (including those used within tools/github/before_install.sh)
+          MINIMUM_REQUIREMENTS: ${{ matrix.MINIMUM_REQUIREMENTS }}
+          PIP_FLAGS: ${{ matrix.PIP_FLAGS }}
+          INSTALL_FROM_SDIST: ${{ matrix.INSTALL_FROM_SDIST }}
+        run: |
+          set -ex
+          source tools/github/before_install.sh
+          set -ex
+          if [[ $INSTALL_FROM_SDIST == "1" ]]; then
+              python -m build --no-isolation --skip-dependency-check --sdist .
+              pip install dist/scikit_image-*.tar.gz
+          else
+              pip install -vv --no-build-isolation .;
+          fi
+
+      - name: Run tests
+        env:
+          # Set variables here that are used by the run script below
+          # (including those used within tools/github/script.sh)
+          # Also includes the EAGER_IMPORT and
+          # SKIMAGE_TEST_STRICT_WARNINGS_GLOBAL env vars that skimage checks.
+          PIP_FLAGS: >-
+            --extra-index-url
+            https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+            --upgrade
+            --pre
+          MINIMUM_REQUIREMENTS: [0]
+          BUILD_DOCS: [1]
+          PYTHONOPTIMIZE: [0]
+          TEST_EXAMPLES: [0]
+          OPTIONAL_DEPS: [0]
+          OPTIONS_NAME: ["default"]
+          INSTALL_FROM_SDIST: [0]
+          WITHOUT_POOCH: [0]
+          SKIMAGE_TEST_STRICT_WARNINGS_GLOBAL: [0]
+          EAGER_IMPORT: [0]
+        run: |
+          source tools/github/script.sh

.github/workflows/nightly-wheel-build.yml

@@ -0,0 +1,32 @@
+name: Nightly Wheel builder
+on:
+  workflow_dispatch:
+  schedule:
+    # this cron is ran every Sunday at midnight UTC
+    - cron: "0 0 * * 0"
+env:
+  CIBW_BUILD_VERBOSITY: 2
+  CIBW_TEST_REQUIRES: "-r requirements/default.txt -r requirements/test.txt"
+  CIBW_TEST_COMMAND: pytest --pyargs skimage
+  CIBW_ENVIRONMENT: PIP_PREFER_BINARY=1
+
+permissions:
+  contents: read
+
+jobs:
+  call-workflow-build-wheels:
+    uses: ./.github/workflows/build_wheels.yml
+
+  upload_anaconda:
+    permissions:
+      contents: write # for softprops/action-gh-release to create GitHub release
+    name: Release
+    needs: call-workflow-build-wheels
+    if: github.repository_owner == 'scikit-image' && startsWith(github.ref, 'refs/tags/v') && always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload wheel
+        uses: scientific-python/upload-nightly-action@main
+        with:
+          artifacts_path: dist
+          anaconda_nightly_upload_token: ${{secrets.UPLOAD_TOKEN}}

.github/workflows/tests.yml

@@ -45,14 +45,10 @@ jobs:
             OPTIONS_NAME: "mini-req-optional-deps"
           - platform_id: manylinux_x86_64
             python-version: 3.11
-            PIP_FLAGS: >-
-              --extra-index-url
-              https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
-              --upgrade
-              --pre
+            PIP_FLAGS: "--pre"
             SKIMAGE_TEST_STRICT_WARNINGS_GLOBAL: 1
-            # test nightly wheels
-            OPTIONS_NAME: "nightly"
+            # test prereleases
+            OPTIONS_NAME: "pre"
           - platform_id: manylinux_x86_64
             python-version: 3.9
             BUILD_DOCS: 1