Testing Data Tagging (#1339)

### What kind of change does this PR introduce?

* Adds some checks when running `pytest` to ensure that `xclim-testdata`
branches used to test against `xclim` are compatible.
* Sets the `main.yml` CI workflow to use tagged commits from
`xclim-testdata`
* Adds a new CI workflow to check whether the tag found in `main.yml` is
consistent with the latest tag in `xclim-testdata`, and posts comments
and/or fails the build if there is disagreement. This is only triggered
if changes are made to `main.yml`
* Updates the documentation on how best to update `xclim-testdata`.

### Does this PR introduce a breaking change?

Yes. `XCLIM_TESTDATA_BRANCH` now points to tagged testing data commits
on CI.

.github/workflows/bump-version.yml

@@ -1,4 +1,4 @@
-name: "Bump Patch Version"
+name: Bump Patch Version
 
 on:
   push:
@@ -26,7 +26,8 @@ on:
 
 jobs:
   bump_patch_version:
-    runs-on: ubuntu-22.04
+    name: Bumpversion Patch
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
@@ -39,12 +40,15 @@ jobs:
           git config --local user.email "bumpversion[bot]@ouranos.ca"
           git config --local user.name "bumpversion[bot]"
       - name: Current Version
-        run: echo "current_version=$(grep -E '__version__'  xclim/__init__.py | cut -d ' ' -f3)"
+        run: |
+          CURRENT_VERSION="$(grep -E '__version__'  xclim/__init__.py | cut -d ' ' -f3)"
+          echo "current_version=${CURRENT_VERSION}"
       - name: Bump Patch Version
         run: |
           pip install bump2version
           echo "running `bump2version patch`"
-          echo "new_version=$(grep -E '__version__'  xclim/__init__.py | cut -d ' ' -f3)"
+          NEW_VERSION="$(grep -E '__version__'  xclim/__init__.py | cut -d ' ' -f3)"
+          echo "new_version=${NEW_VERSION}"
       - name: Push Changes
         uses: ad-m/github-push-action@master
         with:

.github/workflows/codeql-analysis.yml

@@ -1,4 +1,4 @@
-name: "CodeQL Scan"
+name: CodeQL Scan
 
 on:
   push:
@@ -23,7 +23,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     permissions:
       actions: read
       contents: read

.github/workflows/first_pull_request.yml

@@ -1,11 +1,14 @@
+name: First Pull Request
+
 on:
   pull_request_target:
     types:
       - opened
 
 jobs:
   welcome:
-    runs-on: ubuntu-22.04
+    name: Welcome
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/github-script@v6
         with:

.github/workflows/label.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   label:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/labeler@v4
       with:

.github/workflows/label_on_approval.yml

@@ -1,4 +1,4 @@
-name: "Label Pull Request on Pull Request review approval"
+name: Label Pull Request on Pull Request review approval
 
 on:
   pull_request_review:
@@ -7,8 +7,9 @@ on:
 
 jobs:
   approved:
+    name: Label on Approval
     if: github.event.review.state == 'approved'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/github-script@v6
         with:

.github/workflows/main.yml

@@ -20,7 +20,7 @@ on:
       - submitted
 
 env:
-  XCLIM_TESTDATA_BRANCH: main
+  XCLIM_TESTDATA_BRANCH: v0.41.0
 
 jobs:
   black:

.github/workflows/publish-pypi.yml

@@ -1,4 +1,4 @@
-name: "Publish Python 🐍 distributions 📦 to PyPI"
+name: Publish Python 🐍 distributions 📦 to PyPI
 
 on:
   release:
@@ -8,7 +8,7 @@ on:
 jobs:
   build-n-publish-pypi:
     name: Build and publish Python 🐍 distributions 📦 to PyPI
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python3

.github/workflows/tag-testpypi.yml

@@ -1,4 +1,4 @@
-name: "Publish Python 🐍 distributions 📦 to TestPyPI"
+name: Publish Python 🐍 distributions 📦 to TestPyPI
 
 on:
   push:
@@ -8,7 +8,7 @@ on:
 jobs:
   build-n-publish-testpypi:
     name: Build and publish Python 🐍 distributions 📦 to TestPyPI
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python3

.github/workflows/testdata_version.yml

@@ -0,0 +1,70 @@
+name: Verify Testing Data
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    paths:
+      - .github/workflows/main.yml
+
+jobs:
+  use-latest-tag:
+    name: Check Latest xclim-testdata Tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Find xclim-testdata Tag and CI Testing Branch
+        run: |
+          XCLIM_TESTDATA_TAG="$( \
+          git -c 'versionsort.suffix=-' \
+          ls-remote --exit-code --refs --sort='version:refname' --tags https://github.com/Ouranosinc/xclim-testdata '*.*.*' \
+          | tail --lines=1 \
+          | cut --delimiter='/' --fields=3)"
+          echo "XCLIM_TESTDATA_TAG=${XCLIM_TESTDATA_TAG}" >> $GITHUB_ENV
+          XCLIM_TESTDATA_BRANCH="$(grep -E "XCLIM_TESTDATA_BRANCH" .github/workflows/main.yml | cut -d ' ' -f4)"
+          echo "XCLIM_TESTDATA_BRANCH=${XCLIM_TESTDATA_BRANCH}" >> $GITHUB_ENV
+      - name: Report Versions Found
+        run: |
+          echo "Latest xclim-testdata tag: ${{ env.XCLIM_TESTDATA_TAG }}"
+          echo "Tag for xclim-testdata in CI: ${{ env.XCLIM_TESTDATA_BRANCH }}"
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: It appears that this PR modifies the `XCLIM_TESTDATA_BRANCH` environment variable
+      - name: Compare Versions
+        if: ${{( env.XCLIM_TESTDATA_TAG != env.XCLIM_TESTDATA_BRANCH )}}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.setFailed('Configured `xclim-testdata` tag is not `latest`.')
+      - name: Update Failure Comment
+        if: ${{ failure() }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            > **Warning**
+            > It appears that this PR modifies the `XCLIM_TESTDATA_BRANCH` environment variable to a tag that is not the latest in the `Ouranosinc/xclim-testdata` repository.
+
+            Please be sure to modify this value to match the most recent tag (`${{ env.XCLIM_TESTDATA_TAG }}`) before merging.
+
+            If this PR depends on changes in a new testing dataset branch, be sure to tag a new version of `Ouranosinc/xclim-testdata` with your changes merged to `main`.
+          edit-mode: replace
+      - name: Update Success Comment
+        if: ${{ success() }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            > **Note**
+            > It appears that this PR modifies the `XCLIM_TESTDATA_BRANCH` environment variable to the most recent tag (`${{ env.XCLIM_TESTDATA_TAG }}`).
+
+            No further action is required.
+          edit-mode: replace

.github/workflows/upstream.yml

@@ -1,4 +1,4 @@
-name: xclim Upstream dependencies
+name: Test Upstream Dependencies
 on:
   push:
     branches:

CHANGES.rst

@@ -6,6 +6,10 @@ v0.42.0 (unreleased)
 --------------------
 Contributors to this version: Trevor James Smith (:user:`Zeitsperre`), Juliette Lavoie (:user:`juliettelavoie`), Éric Dupuis (:user:`coxipi`).
 
+Announcements
+^^^^^^^^^^^^^
+* `xclim` now supports testing against tagged versions of `Ouranosinc/xclim-testdata <https://github.com/Ouranosinc/xclim-testdata>`_ in order to support older versions of `xclim`. For more information, see the `Contributing Guide <https://xclim.readthedocs.io/en/stable/contributing.html>`_ for more details. (:pull:`1339`).
+
 New features and enhancements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 * Two previously private functions for selecting a day of year in a time series when performing calendar conversions are now exposed. (:issue:`1305`, :pull:`1317`). New functions are:

CONTRIBUTING.rst

@@ -4,8 +4,7 @@
 Contributing
 ============
 
-Contributions are welcome, and they are greatly appreciated! Every little bit
-helps, and credit will always be given.
+Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given.
 
 You can contribute in many ways:
 
@@ -15,7 +14,7 @@ Types of Contributions
 Implement Features, Indices or Indicators
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-xclim's structure makes it easy to create and register new user-defined indices and indicators.
+`xclim`'s structure makes it easy to create and register new user-defined indices and indicators.
 For the general implementation of indices and their wrapping into indicators, refer to
 :ref:`notebooks/extendxclim:Extending xclim`  and  :ref:`notebooks/customize:Customizing and controlling xclim`.
 
@@ -181,7 +180,7 @@ Pull Request Guidelines
 Before you submit a pull request, please follow these guidelines:
 
 1. Open an *issue* on our `GitHub repository`_ with your issue that you'd like to fix or feature that you'd like to implement.
-2. Perform the changes, commit and push them either to new a branch within Ouranosinc/xclim or to your personal fork of xclim.
+2. Perform the changes, commit and push them either to new a branch within `Ouranosinc/xclim` or to your personal fork of xclim.
 
 .. warning::
      Try to keep your contributions within the scope of the issue that you are addressing.
@@ -223,9 +222,33 @@ Before you submit a pull request, please follow these guidelines:
      ^^^^^^^^^^^^^^^^
      * Updated the contribution guidelines. (:issue:`868`, :pull:`869`).
 
-   If this is your first contribution to Ouranosinc/xclim, we ask that you also add your name to the `AUTHORS.rst <https://github.com/Ouranosinc/xclim/blob/master/AUTHORS.rst>`_,
+   If this is your first contribution to `Ouranosinc/xclim`, we ask that you also add your name to the `AUTHORS.rst <https://github.com/Ouranosinc/xclim/blob/master/AUTHORS.rst>`_,
    under *Contributors* as well as to the `.zenodo.json <https://github.com/Ouranosinc/xclim/blob/master/.zenodo.json>`_, at the end of the *creators* block.
 
+Updating Testing Data
+~~~~~~~~~~~~~~~~~~~~~
+
+If your code changes require changes to the testing data of `xclim` (i.e.: modifications to existing datasets or new datasets),
+these changes must be made via a Pull Request at https://github.com/Ouranosinc/xclim-testdata.
+
+`xclim` allows for developers to test specific branches/versions of `xclim-testdata` via the `XCLIM_TESTDATA_BRANCH` environment variable, e.g.::
+
+    $ export XCLIM_TESTDATA_BRANCH="my_new_branch_of_testing_data"
+    $ pytest xclim
+
+This will ensure that tests load the testing data from this branch before running.
+
+If you wish to test a specific branch using GitHub CI, this can be set in `.github/workflows/main.yml`:
+
+.. code-block:: yaml
+
+    env:
+      XCLIM_TESTDATA_BRANCH: my_new_branch_of_testing_data
+
+.. warning::
+    In order for a Pull Request to be allowed to merge to main development branch, this variable must match the latest tagged commit name on `Ouranosinc/xclim-testdata`.
+    We suggest merging changed testing data first, tagging a new version of `xclim-testdata`, then re-running tests on your Pull Request at `Ouranosinc/xclim` with the newest tag.
+
 Tips
 ----
 

xclim/testing/tests/conftest.py

@@ -2,7 +2,9 @@
 from __future__ import annotations
 
 import os
+import re
 import shutil
+import warnings
 from functools import partial
 from pathlib import Path
 
@@ -11,8 +13,10 @@
 import pytest
 import xarray as xr
 from filelock import FileLock
+from pkg_resources import parse_version
 
 import xclim
+from xclim import __version__ as __xclim_version__
 from xclim.core import indicator
 from xclim.core.calendar import max_doy
 from xclim.testing.tests.data import (
@@ -27,6 +31,25 @@
 PREFETCH_TESTING_DATA = os.getenv("XCLIM_PREFETCH_TESTING_DATA")
 
 
+if not __xclim_version__.endswith("-dev") and TESTDATA_BRANCH == "main":
+    # This is fine on GitHub Workflows and ReadTheDocs
+    if not os.getenv("CI") and not os.getenv("READTHEDOCS"):
+        warnings.warn(
+            f'`xclim` {__xclim_version__} is running tests against the "main" branch of `Ouranosinc/xclim-testdata`. '
+            "It is possible that changes in xclim-testdata may be incompatible with tests in this version. "
+            "Please be sure to check https://github.com/Ouranosinc/xclim-testdata for more information.",
+            UserWarning,
+        )
+
+if re.match(r"^v\d+\.\d+\.\d+", TESTDATA_BRANCH):
+    if parse_version(TESTDATA_BRANCH) > parse_version(__xclim_version__):
+        warnings.warn(
+            f"`xclim` version ({__xclim_version__}) predates `xclim-testdata` version ({TESTDATA_BRANCH}). "
+            f"It is very likely that the testing data is incompatible with this build of `xclim`.",
+            UserWarning,
+        )
+
+
 @pytest.fixture
 def tmp_netcdf_filename(tmpdir) -> Path:
     yield Path(tmpdir).joinpath("testfile.nc")

xclim/testing/utils.py

@@ -26,7 +26,6 @@
 import pandas as pd
 from xarray import Dataset
 from xarray import open_dataset as _open_dataset
-from yaml import safe_dump, safe_load
 
 _xclim_deps = [
     "xclim",