From 9a8b2f4541bbbedfc3e9d2306484e856d4c334cf Mon Sep 17 00:00:00 2001
From: Elias <110238618+ESadek-MO@users.noreply.github.com>
Date: Mon, 15 Apr 2024 16:27:25 +0100
Subject: [PATCH] Convert tests and doctests from Cirrus to GHA (#415)

* created ci-tests

* added caches
---
 .cirrus.yml                                   | 130 -----------------
 .github/workflows/ci-tests.yml                | 131 ++++++++++++++++++
 .../composite/cartopy-cache/action.yml        |  41 ++++++
 .../composite/conda-env-cache/action.yml      |  35 +++++
 .../composite/conda-pkg-cache/action.yml      |  22 +++
 .../composite/iris-data-cache/action.yml      |  30 ++++
 .../workflows/composite/nox-cache/action.yml  |  22 +++
 7 files changed, 281 insertions(+), 130 deletions(-)
 delete mode 100644 .cirrus.yml
 create mode 100644 .github/workflows/ci-tests.yml
 create mode 100644 .github/workflows/composite/cartopy-cache/action.yml
 create mode 100644 .github/workflows/composite/conda-env-cache/action.yml
 create mode 100644 .github/workflows/composite/conda-pkg-cache/action.yml
 create mode 100644 .github/workflows/composite/iris-data-cache/action.yml
 create mode 100644 .github/workflows/composite/nox-cache/action.yml

diff --git a/.cirrus.yml b/.cirrus.yml
deleted file mode 100644
index 1de7c187..00000000
--- a/.cirrus.yml
+++ /dev/null
@@ -1,130 +0,0 @@
-# Reference:
-#   - https://cirrus-ci.org/guide/writing-tasks/
-#   - https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
-#   - https://cirrus-ci.org/guide/linux/
-#   - https://cirrus-ci.org/guide/macOS/
-#   - https://cirrus-ci.org/guide/windows/
-#   - https://hub.docker.com/_/gcc/
-#   - https://hub.docker.com/_/python/
-
-#
-# Global defaults.
-#
-container:
-  image: python:3.11
-  cpu: 2
-  memory: 4G
-
-env:
-  # Maximum cache period (in weeks) before forcing a new cache upload.
-  CACHE_PERIOD: "2"
-  # Increment the build number to force new conda cache upload.
-  CONDA_CACHE_BUILD: "98"
-  # Increment the build number to force new nox cache upload.
-  NOX_CACHE_BUILD: "98"
-  # Increment the build number to force new pip cache upload.
-  PIP_CACHE_BUILD: "98"
-  # Pip packages to be upgraded/installed.
-  PIP_CACHE_PACKAGES: "pip setuptools wheel nox pyyaml"
-  # Base directory for the iris-test-data.
-  IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data
-  # Tagged release version of the iris-test-data.
-  IRIS_TEST_DATA_REF: "2.23"
-
-#
-# Linting
-# [2021-03-01] jamesp: Currently not black compliant.
-#                      GitHub issue open #251
-lint_task:
-  auto_cancellation: true
-  name: "${CIRRUS_OS}: flake8" # and black
-  pip_cache:
-    folder: ~/.cache/pip
-    fingerprint_script:
-      - echo "${CIRRUS_TASK_NAME}"
-      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${PIP_CACHE_BUILD} ${PIP_CACHE_PACKAGES}"
-  lint_script:
-    - pip list
-    - python -m pip install --retries 3 --upgrade ${PIP_CACHE_PACKAGES}
-    - pip list
-    - nox --session flake8
-#    - nox --session black
-
-
-#
-# Testing (Linux)
-#
-linux_task_template: &LINUX_TASK_TEMPLATE
-  auto_cancellation: true
-  env:
-    PATH: ${HOME}/miniconda/bin:${PATH}
-
-  container:
-    image: gcc:latest
-    cpu: 6
-    memory: 8G
-  conda_cache:
-    folder: ${HOME}/miniconda
-    fingerprint_script:
-      - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-      - echo "${CIRRUS_OS} $(sha256sum miniconda.sh)"
-      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CONDA_CACHE_BUILD}"
-      - uname -r
-    populate_script:
-      - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)"
-      - bash miniconda.sh -b -p ${HOME}/miniconda
-      - conda config --set always_yes yes --set changeps1 no
-      - conda config --set show_channel_urls True
-      - conda config --add channels conda-forge
-      - conda update --quiet --name base conda
-      - conda install --quiet --name base nox pip coverage pyyaml
-  nox_cache:
-    folder: ${CIRRUS_WORKING_DIR}/.nox
-    reupload_on_changes: true
-    fingerprint_script:
-      - echo "${CIRRUS_TASK_NAME}"
-      - echo "${NOX_CACHE_BUILD}"
-      - echo "IRIS ${IRIS_SOURCE}"
-  data_cache:
-    folder: ${IRIS_TEST_DATA_DIR}
-    fingerprint_script:
-      - echo "${IRIS_TEST_DATA_REF}"
-    populate_script:
-      - wget --quiet https://github.com/SciTools/iris-test-data/archive/v${IRIS_TEST_DATA_REF}.zip -O iris-test-data.zip
-      - unzip -q iris-test-data.zip
-      - mv iris-test-data-$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//") ${IRIS_TEST_DATA_DIR}
-
-tests_task:
-  matrix:
-    - name: "${CIRRUS_OS} tests: iris=conda-release python=3.9"
-      env:
-        PY_VER: 3.9
-        IRIS_SOURCE: "conda-forge"
-    - name: "${CIRRUS_OS} tests: iris=conda-release python=3.10"
-      env:
-        PY_VER: 3.10
-        IRIS_SOURCE: "conda-forge"
-    - name: "${CIRRUS_OS} tests: iris=conda-release python=3.11"
-      env:
-        PY_VER: 3.11
-        IRIS_SOURCE: "conda-forge"
-    - name: "${CIRRUS_OS} tests: iris=repo-main python=3.11"
-      env:
-        PY_VER: 3.11
-        IRIS_SOURCE: "source"
-
-  << : *LINUX_TASK_TEMPLATE
-  tests_script:
-    - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)"
-    - nox --session tests -- --test-data-dir ${IRIS_TEST_DATA_DIR}/test_data
-
-# [2021-06-07] lbdreyer: Doctests are failing, see Github Issue #274
-#doctest_task:
-#  matrix:
-#    env:
-#      PY_VER: 3.7
-#  name: "${CIRRUS_OS}: py${PY_VER} tests"
-#  << : *LINUX_TASK_TEMPLATE
-#  tests_script:
-#    - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)"
-#    - nox --session doctest -- --test-data-dir ${IRIS_TEST_DATA_DIR}/test_data
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
new file mode 100644
index 00000000..47c49c5c
--- /dev/null
+++ b/.github/workflows/ci-tests.yml
@@ -0,0 +1,131 @@
+# reference:
+#   - https://github.com/actions/cache
+#   - https://github.com/actions/checkout
+#   - https://github.com/marketplace/actions/setup-miniconda
+
+name: ci-tests
+
+on:
+  push:
+    branches:
+      - "main"
+      - "v*x"
+    tags:
+      - "v*"
+  pull_request:
+    branches:
+      - "*"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    name: "${{ matrix.session }} (py${{ matrix.python-version }} ${{ matrix.os }}) [${{ matrix.iris-source }}]"
+
+    runs-on: ${{ matrix.os }}
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        python-version: ["3.11"]
+        session: ["doctest"]
+        # conda-forge doesn't fit within nose char limits, change with pytest
+        iris-source : ["condaforge"]
+        include:
+          - os: "ubuntu-latest"
+            python-version: "3.11"
+            session: "tests"
+            iris-source: "condaforge"
+          - os: "ubuntu-latest"
+            python-version: "3.11"
+            session: "tests"
+            iris-source: "source"
+          - os: "ubuntu-latest"
+            python-version: "3.10"
+            session: "tests"
+            iris-source: "condaforge"
+          - os: "ubuntu-latest"
+            python-version: "3.9"
+            session: "tests"
+            iris-source: "condaforge"
+
+    env:
+      IRIS_TEST_DATA_VERSION: "2.24"
+      ENV_NAME: "ci-tests"
+
+    steps:
+      - name: "checkout"
+        uses: actions/checkout@v4
+
+      - name: "environment configure"
+        env:
+          # Maximum cache period (in weeks) before forcing a cache refresh.
+          CACHE_WEEKS: 2
+        run: |
+          echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV}
+          echo "LOCK_FILE=requirements/locks/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV}
+
+      - name: "data cache"
+        uses: ./.github/workflows/composite/iris-data-cache
+        with:
+          cache_build: 0
+          env_name: ${{ env.ENV_NAME }}
+          version: ${{ env.IRIS_TEST_DATA_VERSION }}
+
+      - name: "conda package cache"
+        uses: ./.github/workflows/composite/conda-pkg-cache
+        with:
+          cache_build: 0
+          cache_period: ${{ env.CACHE_PERIOD }}
+          env_name: ${{ env.ENV_NAME }}
+
+      - name: "conda install"
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          channels: conda-forge,defaults
+          activate-environment: ${{ env.ENV_NAME }}
+          auto-update-conda: false
+          use-only-tar-bz2: true
+
+      - name: "conda environment cache"
+        uses: ./.github/workflows/composite/conda-env-cache
+        with:
+          cache_build: 0
+          cache_period: ${{ env.CACHE_PERIOD }}
+          env_name: ${{ env.ENV_NAME }}
+          install_packages: "cartopy nox pip"
+
+      - name: "conda info"
+        run: |
+          conda info
+          conda list
+
+      - name: "cartopy cache"
+        uses: ./.github/workflows/composite/cartopy-cache
+        with:
+          cache_build: 0
+          cache_period: ${{ env.CACHE_PERIOD }}
+          env_name: ${{ env.ENV_NAME }}
+
+      - name: "nox cache"
+        uses: ./.github/workflows/composite/nox-cache
+        with:
+          cache_build: 0
+          env_name: ${{ env.ENV_NAME }}
+          lock_file: ${{ env.LOCK_FILE }}
+
+      - name: "iris ${{ matrix.session }}"
+        env:
+          PY_VER: ${{ matrix.python-version }}
+          IRIS_SOURCE: ${{ matrix.iris-source }}
+        run: |
+          nox --session ${{ matrix.session }} -- --test-data-dir ${HOME}/iris-test-data/test_data
\ No newline at end of file
diff --git a/.github/workflows/composite/cartopy-cache/action.yml b/.github/workflows/composite/cartopy-cache/action.yml
new file mode 100644
index 00000000..e41f575a
--- /dev/null
+++ b/.github/workflows/composite/cartopy-cache/action.yml
@@ -0,0 +1,41 @@
+name: "cartopy-cache"
+description: "create and cache cartopy assets"
+
+#
+# Assumes the environment contains the following variables:
+#   - CONDA
+#
+inputs:
+  cache_build:
+    description: "conda environment cache build number"
+    required: false
+    default: "0"
+  cache_period:
+    description: "conda environment cache timestamp"
+    required: true
+  env_name:
+    description: "environment name"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/cache@v3
+      id: cartopy-cache
+      with:
+        path: ~/.local/share/cartopy
+        key: ${{ runner.os }}-cartopy-${{ inputs.env_name }}-p${{ inputs.cache_period }}-b${{ inputs.cache_build }}
+
+    - if: steps.cartopy-cache.outputs.cache-hit != 'true'
+      env:
+        CARTOPY_SHARE_DIR: ~/.local/share/cartopy
+        CARTOPY_FEATURE: https://raw.githubusercontent.com/SciTools/cartopy/v0.20.0/tools/cartopy_feature_download.py
+      shell: bash
+      run: |
+        # Require to explicitly activate the environment within the composite action.
+        source ${{ env.CONDA }}/etc/profile.d/conda.sh >/dev/null 2>&1
+        conda activate ${{ inputs.env_name }}
+        wget --quiet ${CARTOPY_FEATURE}
+        mkdir -p ${CARTOPY_SHARE_DIR}
+        # Requires a pre-installed version of cartopy within the environment.
+        python cartopy_feature_download.py physical --output ${CARTOPY_SHARE_DIR} --no-warn
\ No newline at end of file
diff --git a/.github/workflows/composite/conda-env-cache/action.yml b/.github/workflows/composite/conda-env-cache/action.yml
new file mode 100644
index 00000000..7a32a792
--- /dev/null
+++ b/.github/workflows/composite/conda-env-cache/action.yml
@@ -0,0 +1,35 @@
+name: "conda-env-cache"
+description: "create and cache the conda environment"
+
+#
+# Assumes the environment contains the following variables:
+#   - CONDA
+#
+inputs:
+  cache_build:
+    description: "conda environment cache build number"
+    required: false
+    default: "0"
+  cache_period:
+    description: "conda environment cache timestamp"
+    required: true
+  env_name:
+    description: "environment name"
+    required: true
+  install_packages:
+    description: "conda packages to install into environment"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/cache@v3
+      id: conda-env-cache
+      with:
+        path: ${{ env.CONDA }}/envs/${{ inputs.env_name }}
+        key: ${{ runner.os }}-conda-env-${{ inputs.env_name }}-p${{ inputs.cache_period }}-b${{ inputs.cache_build }}
+
+    - if: steps.conda-env-cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        conda install --quiet --name ${{ inputs.env_name }} ${{ inputs.install_packages }}
\ No newline at end of file
diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml
new file mode 100644
index 00000000..d06c0941
--- /dev/null
+++ b/.github/workflows/composite/conda-pkg-cache/action.yml
@@ -0,0 +1,22 @@
+name: "conda-pkg-cache"
+description: "cache the conda environment packages"
+
+inputs:
+  cache_build:
+    description: "conda environment cache build number"
+    required: false
+    default: "0"
+  cache_period:
+    description: "conda environment cache timestamp"
+    required: true
+  env_name:
+    description: "environment name"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/cache@v3
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-conda-pkgs-${{ inputs.env_name }}-p${{ inputs.cache_period }}-b${{ inputs.cache_build }}
\ No newline at end of file
diff --git a/.github/workflows/composite/iris-data-cache/action.yml b/.github/workflows/composite/iris-data-cache/action.yml
new file mode 100644
index 00000000..5869ab64
--- /dev/null
+++ b/.github/workflows/composite/iris-data-cache/action.yml
@@ -0,0 +1,30 @@
+name: "iris-data-cache"
+description: "create and cache the iris test data"
+
+inputs:
+  cache_build:
+    description: "data cache build number"
+    required: false
+    default: "0"
+  env_name:
+    description: "environment name"
+    required: true
+  version:
+    description: "iris test data version"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/cache@v3
+      id: data-cache
+      with:
+        path: ~/iris-test-data
+        key: ${{ runner.os }}-iris-test-data-${{ inputs.env_name }}-v${{ inputs.version }}-b${{ inputs.cache_build }}
+
+    - if: steps.data-cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        wget --quiet https://github.com/SciTools/iris-test-data/archive/v${{ inputs.version }}.zip -O iris-test-data.zip
+        unzip -q iris-test-data.zip
+        mv iris-test-data-${{ inputs.version }} ~/iris-test-data
\ No newline at end of file
diff --git a/.github/workflows/composite/nox-cache/action.yml b/.github/workflows/composite/nox-cache/action.yml
new file mode 100644
index 00000000..23447f84
--- /dev/null
+++ b/.github/workflows/composite/nox-cache/action.yml
@@ -0,0 +1,22 @@
+name: "nox cache"
+description: "cache the nox test environments"
+
+inputs:
+  cache_build:
+    description: "nox cache build number"
+    required: false
+    default: "0"
+  env_name:
+    description: "environment name"
+    required: true
+  lock_file:
+    description: "conda-lock environment requirements filename"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/cache@v3
+      with:
+        path: ${{ github.workspace }}/.nox
+        key: ${{ runner.os }}-nox-${{ inputs.env_name }}-py${{ matrix.python-version }}-b${{ inputs.cache_build }}-${{ hashFiles(inputs.lock_file) }}
\ No newline at end of file