BLD, CI: Use cirrus for building aarch64 wheels

Backport of #22650, #22685, #22690. The travisci builds were failing
for the Python 3.8 builds, so use cirrus instead.

.cirrus.star

@@ -0,0 +1,38 @@
+# The guide to programming cirrus-ci tasks using starlark is found at
+# https://cirrus-ci.org/guide/programming-tasks/
+#
+# In this simple starlark script we simply check conditions for whether
+# a CI run should go ahead. If the conditions are met, then we just
+# return the yaml containing the tasks to be run.
+
+load("cirrus", "env", "fs", "http")
+
+def main(ctx):
+    ######################################################################
+    # Should wheels be built?
+    # Only test on the numpy/numpy repository
+    ######################################################################
+
+    if env.get("CIRRUS_REPO_FULL_NAME") != "numpy/numpy":
+        return []
+
+    # if env.get("CIRRUS_CRON", "") == "nightly":
+    #     return fs.read("ci/cirrus_wheels.yml")
+
+    # Obtain commit message for the event. Unfortunately CIRRUS_CHANGE_MESSAGE
+    # only contains the actual commit message on a non-PR trigger event.
+    # For a PR event it contains the PR title and description.
+    SHA = env.get("CIRRUS_CHANGE_IN_REPO")
+    url = "https://api.github.com/repos/scipy/scipy/git/commits/" + SHA
+    dct = http.get(url).json()
+    # if "[wheel build]" in dct["message"]:
+    #     return fs.read("ci/cirrus_wheels.yml")
+
+    if "[skip cirrus]" in dct["message"] or "[skip ci]" in dct["message"]:
+        return []
+
+    config =  fs.read("tools/ci/cirrus_general.yml")
+
+    # add extra jobs to the cirrus run by += adding to config
+
+    return config

.travis.yml

@@ -50,64 +50,8 @@ jobs:
        - ATLAS=None
        - EXPECT_CPU_FEATURES="VX VXE VXE2"
 
-    # Wheel builders
-    # Note that Ubuntu focal comes with Python 3.8 and CIBW_BUILD determines
-    # the Python used to build the wheels.
-    - python: "3.8"
-      os: linux
-      arch: arm64
-      virt: vm
-      env:
-        - CIBW_BUILD: cp38-manylinux_aarch64
-        - EXPECT_CPU_FEATURES: "NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM"
-      install: python3 -m pip install cibuildwheel==2.11.2
-      script: |
-        cibuildwheel --output-dir wheelhouse
-        source ./tools/wheels/upload_wheels.sh
-        set_travis_vars
-        set_upload_vars
-        upload_wheels # Will be skipped if not a push/tag/scheduled build
-    - python: "3.8"
-      os: linux
-      arch: arm64
-      virt: vm
-      env:
-        - CIBW_BUILD: cp39-manylinux_aarch64
-      install: python3 -m pip install cibuildwheel==2.11.2
-      script: |
-        cibuildwheel --output-dir wheelhouse
-        source ./tools/wheels/upload_wheels.sh
-        set_travis_vars
-        set_upload_vars
-        upload_wheels # Will be skipped if not a push/tag/scheduled build
-    - python: "3.8"
-      os: linux
-      arch: arm64
-      virt: vm
-      env:
-        - CIBW_BUILD: cp310-manylinux_aarch64
-      install: python3 -m pip install cibuildwheel==2.11.2
-      script: |
-        cibuildwheel --output-dir wheelhouse
-        source ./tools/wheels/upload_wheels.sh
-        set_travis_vars
-        set_upload_vars
-        upload_wheels # Will be skipped if not a push/tag/scheduled build
-    - python: "3.8"
-      os: linux
-      arch: arm64
-      virt: vm
-      env:
-        - CIBW_BUILD: cp311-manylinux_aarch64
-      install: python3 -m pip install cibuildwheel==2.11.2
-      script: |
-        cibuildwheel --output-dir wheelhouse
-        source ./tools/wheels/upload_wheels.sh
-        set_travis_vars
-        set_upload_vars
-        upload_wheels # Will be skipped if not a push/tag/scheduled build
 before_install:
   - ./tools/travis-before-install.sh
 
 script:
-  - travis_wait 30 ./tools/travis-test.sh
+  - ./tools/travis-test.sh

tools/ci/cirrus_general.yml

@@ -0,0 +1,93 @@
+build_and_store_wheels: &BUILD_AND_STORE_WHEELS
+  install_cibuildwheel_script:
+    - python -m pip install cibuildwheel==2.11.2
+  cibuildwheel_script:
+    - cibuildwheel
+  wheels_artifacts:
+    path: "wheelhouse/*"
+
+
+######################################################################
+# Build linux_aarch64 natively
+######################################################################
+
+linux_aarch64_task:
+  compute_engine_instance:
+    image_project: cirrus-images
+    image: family/docker-builder-arm64
+    architecture: arm64
+    platform: linux
+    cpu: 2
+    memory: 8G
+  matrix:
+    # build in a matrix because building and testing all four wheels in a
+    # single task takes longer than 60 mins (the default time limit for a
+    # cirrus-ci task).
+    - env:
+        CIBW_BUILD: cp38-*
+        EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM
+    - env:
+        CIBW_BUILD: cp39-*
+    - env:
+        CIBW_BUILD: cp310-*
+    - env:
+        CIBW_BUILD: cp311-*
+
+  build_script: |
+    apt install -y python3-venv python-is-python3 gfortran libatlas-base-dev libgfortran5 eatmydata
+    git fetch origin
+    ./tools/travis-before-install.sh
+    which python
+    echo $CIRRUS_CHANGE_MESSAGE
+  <<: *BUILD_AND_STORE_WHEELS
+
+
+######################################################################
+# Upload all wheels
+######################################################################
+
+wheels_upload_task:
+  # Artifacts don't seem to be persistent from task to task.
+  # Rather than upload wheels at the end of each cibuildwheel run we do a
+  # final upload here. This is because a run may be on different OS for
+  # which bash, etc, may not be present.
+  depends_on:
+    - linux_aarch64
+  compute_engine_instance:
+    image_project: cirrus-images
+    image: family/docker-builder
+    platform: linux
+    cpu: 1
+
+  env:
+    NUMPY_STAGING_UPLOAD_TOKEN: ENCRYPTED[!5a69522ae0c2af9edb2bc1cdfeaca6292fb3666d9ecd82dca0615921834a6ce3b702352835d8bde4ea2a9ed5ef8424ac!]
+    NUMPY_NIGHTLY_UPLOAD_TOKEN: ENCRYPTED[!196422e6c3419a3b1d79815e1026094a215cb0f346fe34ed0f9d3ca1c19339df7398d04556491b1e0420fc1fe3713289!]
+
+  upload_script: |
+    apt-get install -y python3-venv python-is-python3 curl
+    export IS_SCHEDULE_DISPATCH="false"
+    export IS_PUSH="false"
+
+    # cron job
+    if [[ "$CIRRUS_CRON" == "weekly" ]]; then
+      export IS_SCHEDULE_DISPATCH="true"
+    fi
+
+    if [[ "$CIRRUS_TAG" == "v*" ]]; then
+      export IS_PUSH="true"
+    fi
+    if [[ "$CIRRUS_BUILD_SOURCE" == "api" && "$CIRRUS_COMMIT_MESSAGE" == "API build for null" ]]; then      export IS_SCHEDULE_DISPATCH="true"
+    fi
+
+
+    # The name of the zip file is derived from the `wheels_artifact` line.
+    # If you change the artifact line to `myfile_artifact` then it would be
+    # called myfile.zip
+
+    curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/wheels.zip --output wheels.zip
+    unzip wheels.zip
+
+    source ./tools/wheels/upload_wheels.sh
+    # set_travis_vars
+    set_upload_vars
+    upload_wheels # Will be skipped if not a push/tag/scheduled build

tools/wheels/upload_wheels.sh

@@ -9,9 +9,6 @@ set_travis_vars() {
     fi
     if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then
       IS_SCHEDULE_DISPATCH="true"
-    elif [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then
-      # Manual CI run, so upload
-      IS_SCHEDULE_DISPATCH="true"
     else
       IS_SCHEDULE_DISPATCH="false"
     fi
@@ -37,7 +34,7 @@ set_upload_vars() {
 upload_wheels() {
     echo ${PWD}
     if [[ ${ANACONDA_UPLOAD} == true ]]; then
-        if [ -z ${TOKEN} ]; then
+        if [[ -z ${TOKEN} ]]; then
             echo no token set, not uploading
         else
             python -m pip install \