Add caching to CI environment setup (#320)

* Add conda env caching as in GeoUtils

* Fix order of pop to remove pip dict from conda package list

* Fix pip dependency listing

* Linting

* Fix import of xdem.misc

* Add one unset variable

* Small change to check if caching works when CI reruns

.github/get_yml_env_nopy.py

@@ -0,0 +1,53 @@
+import argparse
+
+import yaml  # type: ignore
+
+
+def environment_yml_nopy(fn_env: str, print_dep: str = "both") -> None:
+    """
+    List dependencies in environment.yml without python version for setup of continuous integration.
+
+    :param fn_env: Filename path to environment.yml
+    :param print_dep: Whether to print conda differences "conda", pip differences "pip" or both.
+    """
+
+    # Load the yml as dictionary
+    yaml_env = yaml.safe_load(open(fn_env))
+    conda_dep_env = list(yaml_env["dependencies"])
+
+    if isinstance(conda_dep_env[-1], dict):
+        pip_dep_env = list(conda_dep_env.pop()["pip"])
+    else:
+        pip_dep_env = ["None"]
+
+    conda_dep_env_without_python = [dep for dep in conda_dep_env if "python" not in dep]
+
+    # Join the lists
+    joined_list_conda_dep = " ".join(conda_dep_env_without_python)
+    joined_list_pip_dep = " ".join(pip_dep_env)
+
+    # Print to be captured in bash
+    if print_dep == "both":
+        print(joined_list_conda_dep)
+        print(joined_list_pip_dep)
+    elif print_dep == "conda":
+        print(joined_list_conda_dep)
+    elif print_dep == "pip":
+        print(joined_list_pip_dep)
+    else:
+        raise ValueError('The argument "print_dep" can only be "conda", "pip" or "both".')
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Get environment list without python version.")
+    parser.add_argument("fn_env", metavar="fn_env", type=str, help="Path to the environment file.")
+    parser.add_argument(
+        "--p",
+        dest="print_dep",
+        default="both",
+        type=str,
+        help="Whether to print conda dependencies, pip ones, or both.",
+    )
+
+    args = parser.parse_args()
+    environment_yml_nopy(fn_env=args.fn_env, print_dep=args.print_dep)

.github/workflows/python-package.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   test:
-    name: ${{ matrix.os }}, python ${{ matrix.python-version}}
+    name: ${{ matrix.os }}, python ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -27,37 +27,68 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    # Set up the conda-forge environment with mamba
-    - name: Set up Python ${{ matrix.python-version }} and base environment
+    # We initiate the environment empty, and check if a key for this environment doesn't already exist in the cache
+    - name: Initiate empty environment
       uses: conda-incubator/setup-miniconda@v2
       with:
+        miniforge-variant: Mambaforge
+        miniforge-version: latest
         auto-update-conda: true
-        python-version: ${{ matrix.python-version }}
-        mamba-version: "*"
-        channels: conda-forge
+        use-mamba: true
         channel-priority: strict
-        environment-file: environment.yml
-        activate-environment: xdem
+        activate-environment: xdem-dev
 
-    - name: Install project
+    - name: Get month for resetting cache
+      id: get-date
+      run: echo "::set-output name=month::$(/bin/date -u '+%Y%m')"
+      shell: bash
+
+    - name: Cache conda env
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CONDA }}/envs
+        key: conda-${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get-date.outputs.month }}-${{ hashFiles('dev-environment.yml') }}-${{ env.CACHE_NUMBER }}
+      env:
+        CACHE_NUMBER: 0 # Increase this value to reset cache if environment.yml has not changed
+      id: cache
+
+    # The trick below is necessary because the generic environment file does not specify a Python version, and only
+    # "conda env update" can be used to update with an environment file, which upgrades the Python version
+    - name: Install base environment with a fixed Python version
+      if: steps.cache.outputs.cache-hit != 'true'
       run: |
-        pip install --no-dependencies --editable .
+        mamba install pyyaml python=${{ matrix.python-version }}
+        pkgs_conda_base=`python .github/get_yml_env_nopy.py "environment.yml" --p "conda"`
+        pkgs_pip_base=`python .github/get_yml_env_nopy.py "environment.yml" --p "pip"`
+        mamba install python=${{ matrix.python-version }} $pkgs_conda_base
+        if [[ "$pkgs_pip_base" != "None" ]]; then
+          pip install $pkgs_pip_base
+        fi
+
+    - name: Install project
+      run: pip install -e . --no-dependencies
 
     # This steps allows us to check the "import xdem" with the base environment provided to users, before adding
     # development-specific dependencies by differencing the env and dev-env yml files
-    - name: Check normal environment import
+    - name: Check import works with base environment
       run: |
         # We unset the PROJ_LIB environment variable to make PROJ work on Windows
         unset PROJ_LIB
         python -c "import xdem"
 
-    - name: Update environment with dev dependencies
+    # This time, the trick below is necessary because: 1/ "conda update" does not support a file -f as argument
+    # and also 2/ "conda env update" does not support --freeze-installed or --no-update-deps
+    - name: Update environment with development packages if cache does not exist
+      if: steps.cache.outputs.cache-hit != 'true'
       run: |
-        mamba env update --name xdem --file dev-environment.yml
-
-    - name: Re-install project
-      run: |
-        pip install -e . --no-dependencies
+        # We unset the PROJ_LIB environment variable to make PROJ work on Windows
+        unset PROJ_LIB
+        pkgs_conda_dev=`python -c "import xdem.misc; xdem.misc.diff_environment_yml('environment.yml', 'dev-environment.yml', 'conda')"`
+        pkgs_pip_dev=`python -c "import xdem.misc; xdem.misc.diff_environment_yml('environment.yml', 'dev-environment.yml', 'pip')"`
+        mamba install $pkgs_conda_dev --freeze-installed
+        if [[ "$pkgs_pip_dev" != "None" ]]; then
+          pip install $pkgs_pip_dev
+        fi
 
     - name: Lint with flake8
       run: |
@@ -66,19 +97,19 @@ jobs:
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
+    - name: Setup pip dependencies
+      run: pip install pytest-cov coveralls coveragepy-lcov
+
     - name: Test with pytest
       run: |
         # We unset the PROJ_LIB environment variable to make PROJ work on Windows
         unset PROJ_LIB
-        pip install pytest-cov coveralls
         pytest -ra --cov=xdem/
 
     # We can skip the conversion step once this PR of pytest is merged: https://github.com/pytest-dev/pytest-cov/pull/536
     # and replace pytest argument by --cov-report=lcov
     - name: Converting coverage to LCOV format
-      run: |
-        pip install coveragepy-lcov
-        coveragepy-lcov --data_file_path .coverage --output_file_path coverage.info
+      run: coveragepy-lcov --data_file_path .coverage --output_file_path coverage.info
 
     - name: Upload coverage to Coveralls
       uses: coverallsapp/github-action@master

xdem/misc.py

@@ -7,6 +7,13 @@
 
 from packaging.version import Version
 
+try:
+    import yaml  # type: ignore
+
+    _has_yaml = True
+except ImportError:
+    _has_yaml = False
+
 try:
     import cv2
 
@@ -116,3 +123,78 @@ def new_func(*args: Any, **kwargs: Any) -> Any:
         return new_func
 
     return deprecator_func
+
+
+def diff_environment_yml(fn_env: str, fn_devenv: str, print_dep: str = "both") -> None:
+    """
+    Compute the difference between environment.yml and dev-environment.yml for setup of continuous integration,
+    while checking that all the dependencies listed in environment.yml are also in dev-environment.yml
+    :param fn_env: Filename path to environment.yml
+    :param fn_devenv: Filename path to dev-environment.yml
+    :param print_dep: Whether to print conda differences "conda", pip differences "pip" or both.
+    """
+
+    if not _has_yaml:
+        raise ValueError("Test dependency needed. Install 'pyyaml'")
+
+    # Load the yml as dictionaries
+    yaml_env = yaml.safe_load(open(fn_env))
+    yaml_devenv = yaml.safe_load(open(fn_devenv))
+
+    # Extract the dependencies values
+    conda_dep_env = yaml_env["dependencies"]
+    conda_dep_devenv = yaml_devenv["dependencies"]
+
+    # Check if there is any pip dependency, if yes pop it from the end of the list
+    if isinstance(conda_dep_devenv[-1], dict):
+        pip_dep_devenv = conda_dep_devenv.pop()["pip"]
+
+        # Check if there is a pip dependency in the normal env as well, if yes pop it also
+        if isinstance(conda_dep_env[-1], dict):
+            pip_dep_env = conda_dep_env.pop()["pip"]
+
+            # The diff below computes the dependencies that are in env but not in dev-env
+            # It should be empty, otherwise we raise an error
+            diff_pip_check = list(set(pip_dep_env) - set(pip_dep_devenv))
+            if len(diff_pip_check) != 0:
+                raise ValueError(
+                    "The following pip dependencies are listed in env but not dev-env: " + ",".join(diff_pip_check)
+                )
+
+            # The diff below computes the dependencies that are in dev-env but not in env, to add during CI
+            diff_pip_dep = list(set(pip_dep_devenv) - set(pip_dep_env))
+
+        # If there is no pip dependency in env, all the ones of dev-env need to be added during CI
+        else:
+            diff_pip_dep = list(pip_dep_devenv["pip"])
+
+    # If there is no pip dependency, we ignore this step
+    else:
+        diff_pip_dep = []
+
+    # If the diff is empty for pip, return a string "None" to read easily in bash
+    if len(diff_pip_dep) == 0:
+        diff_pip_dep = ["None"]
+
+    # We do the same for the conda dependency, first a sanity check that everything that is in env is also in dev-ev
+    diff_conda_check = list(set(conda_dep_env) - set(conda_dep_devenv))
+    if len(diff_conda_check) != 0:
+        raise ValueError("The following dependencies are listed in env but not dev-env: " + ",".join(diff_conda_check))
+
+    # Then the difference to add during CI
+    diff_conda_dep = list(set(conda_dep_devenv) - set(conda_dep_env))
+
+    # Join the lists
+    joined_list_conda_dep = " ".join(diff_conda_dep)
+    joined_list_pip_dep = " ".join(diff_pip_dep)
+
+    # Print to be captured in bash
+    if print_dep == "both":
+        print(joined_list_conda_dep)
+        print(joined_list_pip_dep)
+    elif print_dep == "conda":
+        print(joined_list_conda_dep)
+    elif print_dep == "pip":
+        print(joined_list_pip_dep)
+    else:
+        raise ValueError('The argument "print_dep" can only be "conda", "pip" or "both".')