Add caching (#73)

* Implement caching Signed-off-by: Conor MacBride <conor@macbride.me> * Add documentation Signed-off-by: Conor MacBride <conor@macbride.me> * Add tests Signed-off-by: Conor MacBride <conor@macbride.me> * Update template Signed-off-by: Conor MacBride <conor@macbride.me> * Update test Signed-off-by: Conor MacBride <conor@macbride.me> * Change spec Signed-off-by: Conor MacBride <conor@macbride.me> * Fix typo Signed-off-by: Conor MacBride <conor@macbride.me> * Fix issues Signed-off-by: Conor MacBride <conor@macbride.me> * Second stage for cache verification Signed-off-by: Conor MacBride <conor@macbride.me> * Add docker pull command Signed-off-by: Conor MacBride <conor@macbride.me> * Fix final caching Signed-off-by: Conor MacBride <conor@macbride.me>
OpenAstronomy · Nov 1, 2021 · b273771 · b273771
1 parent 9d1e97f
commit b273771
Show file tree

Hide file tree

Showing 5 changed files with 207 additions and 50 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -4,55 +4,92 @@ variables:
   CIBW_BEFORE_BUILD_LINUX: "yum -y install fftw-devel; git clean -xfd -e wheelhouse"
   CIBW_BEFORE_BUILD_MACOS: "brew install fftw; git clean -xfd -e wheelhouse"
 
-jobs:
-
-- template: run-tox-env.yml
-  parameters:
-    libraries:
-      yum:
-        - openssl-devel
-    envs:
-    - linux: py37-simple-linux
-      name: py37_simple_linux_custom_name
-    - linux32: py37-simple-linux32
-    - linux32: py38-simple-linux32
-    - manylinux: py37-simple-linux
-      manylinux_image: manylinux2014_x86_64
-    - linux: py39-simple-linux
-      docker_image: python:3.9.0rc1-slim-buster
-      docker_python: /usr/local/bin/python
-    - macos: py37-simple-macos
-    - windows: py37-simple-windows
-    - macos: compiler_macos_conda
-
-- template: run-tox-env.yml
-  parameters:
-    mesaopengl: true
-    envs:
-    - macos: py37-opengl-macos
-    - windows: py37-opengl-windows
-
-- template: run-tox-env.yml
-  parameters:
-    envs:
-    - linux: testtoxargs
-      toxargs: --notest
-
-- template: publish.yml
-  parameters:
-    ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-      artifact_project: azure-pipelines-templates
-      artifact_feed: testfeed
-    remove_local_scheme: true
-    libraries:
-      - libfftw3-dev
-    targets:
-      - wheels_linux
-      - wheels_macos
-      - wheels_cp3[78]-win_amd64
-      - wheels_cp37-manylinux_x86_64
-      - wheels_cp37-manylinux_aarch64
-      - sdist
+stages:
+  - stage: StageOneTests
+    displayName: Standard Tests
+    jobs:
+
+    - template: run-tox-env.yml
+      parameters:
+        libraries:
+          yum:
+            - openssl-devel
+        envs:
+        - linux: py37-simple-linux
+          name: py37_simple_linux_custom_name
+        - linux32: py37-simple-linux32
+        - linux32: py38-simple-linux32
+        - manylinux: py37-simple-linux
+          manylinux_image: manylinux2014_x86_64
+        - linux: py39-simple-linux
+          docker_image: python:3.9.0rc1-slim-buster
+          docker_python: /usr/local/bin/python
+        - macos: py37-simple-macos
+        - windows: py37-simple-windows
+        - macos: compiler_macos_conda
+
+    - template: run-tox-env.yml
+      parameters:
+        mesaopengl: true
+        envs:
+        - macos: py37-opengl-macos
+        - windows: py37-opengl-windows
+
+    - template: run-tox-env.yml
+      parameters:
+        envs:
+        - linux: testtoxargs
+          toxargs: --notest
+
+    - template: run-tox-env.yml
+      parameters:
+        cache_dirs:
+        - key: '"data_x" | "$(Build.BuildNumber)"'
+          path: cache_1
+        - key: '"data_y" | "$(Build.BuildNumber)"'
+          path: cache_2
+        envs:
+        - linux: py39-cache-a
+        - linux: py39-simple-linux
+          docker_image: python:3.9.0rc1-slim-buster
+          docker_python: /usr/local/bin/python
+          docker_cache: true
+          cache_dirs: []
+
+    - template: publish.yml
+      parameters:
+        ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+          artifact_project: azure-pipelines-templates
+          artifact_feed: testfeed
+        remove_local_scheme: true
+        libraries:
+          - libfftw3-dev
+        targets:
+          - wheels_linux
+          - wheels_macos
+          - wheels_cp3[78]-win_amd64
+          - wheels_cp37-manylinux_x86_64
+          - wheels_cp37-manylinux_aarch64
+          - sdist
+
+  - stage: StageTwoTests
+    displayName: Cache Verification Tests
+    condition: succeeded()
+    jobs:
+    - template: run-tox-env.yml
+      parameters:
+        cache_dirs:
+        - key: '"data_x" | "$(Build.BuildNumber)"'
+          path: cache_1
+        - key: '"data_y" | "$(Build.BuildNumber)"'
+          path: cache_2
+        envs:
+        - linux: py39-cache-b
+          cache_dirs:
+          - key: '"data_x" | "$(Build.BuildNumber)"'
+            path: cache_3
+          - key: '"data_z" | "$(Build.BuildNumber)"'
+            path: cache_1
 
 # - template: publish.yml
 #   parameters:

diff --git a/docs/run_tox_env.rst b/docs/run_tox_env.rst
@@ -255,6 +255,74 @@ underscore
 Which is why they are not automatically set from the tox env names, as they
 frequently have hyphens in.
 
+Caching
+-------
+Setting the ``cache_dirs`` parameter will cache all files in the specified
+directories. Caches are identified by a specified key. Once a cache is
+created with a particular key, it cannot be updated or replaced.
+The `Azure documentation
+<https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#using-the-cache-task>`__
+contains more information on how Azure manages caching.
+
+A list of caches are defined according to the following specification.
+Each collection of mappings in the sequence under ``cache_dirs`` is passed to
+the ``input`` of a ``Cache@2`` Azure task.
+See the `Azure documentation
+<https://docs.microsoft.com/en-us/azure/devops/pipelines/release/caching?view=azure-devops#using-the-cache-task>`__
+for the full specification of ``input``.
+
+.. code:: yaml
+
+    jobs:
+    - template: run-tox-env.yml@OpenAstronomy
+      parameters:
+        cache_dirs:
+        - key: <cache name>
+          path: <cached directory>
+        - key: <cache name>
+          path: <cached directory>
+      envs:
+      - <os>: <tox env>
+        cache_dirs:
+        - key: <cache name>
+          path: <cached directory>
+        - key: <cache name>
+          path: <cached directory>
+
+
+Azure will look for a cache with the key ``<cache name>``.
+If it exists, it will be restored to the directory ``<cached directory>``.
+If it doesn't exist, at the end of the job the contents of
+``<cached directory>`` will be cached with the key ``<cache name>``
+and will be available for subsequent jobs.
+
+The path specified by ``<cached directory>`` can be an absolute or relative
+path, with relative paths based at ``$(System.DefaultWorkingDirectory)``,
+which is usually the directory containing your package's ``setup.py`` file.
+
+By defining ``cache_dirs`` under ``parameters``, the specified caches will be
+used for all ``envs``. However, if ``cache_dirs`` is specified under a specific
+environment, that environment will *only* use this set of caches.
+
+As an example, to cache pip packages you can set the ``PIP_CACHE_DIR`` environment variable
+and cache this directory. This will ensure that ``pip`` uses this directory as the cache, and
+that it is cached by Azure:
+
+.. code:: yaml
+
+    variables:
+      PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
+
+    jobs:
+    - template: run-tox-env.yml@OpenAstronomy
+      parameters:
+        cache_dirs:
+        - key: 'python | "$(Agent.OS)" | requirements.txt'
+          restoreKeys: |
+            python | "$(Agent.OS)"
+            python
+          path: $(PIP_CACHE_DIR)
+
 
 Docker Jobs
 -----------
@@ -303,9 +371,11 @@ The following example shows all the possible options even though some are redund
           docker_image: python:3.9.0rc1-slim-buster
           docker_name: python39
           docker_python: /usr/local/bin/python
+          docker_cache: true
 
 The options are as follows:
 
 * ``docker_image`` this is the name of the container to be created. It can be any valid argument to ``docker pull``, i.e ``python`` or ``quay.io/pypa/manylinux2010_i686``.
 * ``docker_name`` this is optional as long as ``docker_image`` is a valid container name. If you specify a tag in ``docker_image`` ``:`` and ``/`` will be replaced, so you will not need to specify ``docker_name``. However, if you specify a more complex image you will need to manually specify the container name with ``docker_name``.
 * ``docker_python`` this is the path inside the container to the docker executable.
+* ``docker_cache`` this enables caching of ``docker_image``. This is optional and default is no caching.
diff --git a/run-tox-env.yml b/run-tox-env.yml
@@ -90,9 +90,37 @@ jobs:
           fetchDepth: 999999999
 
         - ${{ if variables.docker_image }}:
+          - ${{ if env['docker_cache'] }}:
+            - task: Cache@2
+              displayName: Processing Docker cache
+              inputs:
+                key: 'docker | "$(Agent.OS)" | "${{ variables.docker_image }}"'
+                path: $(Pipeline.Workspace)/docker
+                cacheHitVar: CACHE_RESTORED
+            - script: |
+                docker load -i $(Pipeline.Workspace)/docker/cache.tar
+              displayName: Docker restore
+              condition: and(not(canceled()), eq(variables.CACHE_RESTORED, 'true'))
+            - script: |
+                docker pull ${{ variables.docker_image }}
+                mkdir -p $(Pipeline.Workspace)/docker
+                docker save -o $(Pipeline.Workspace)/docker/cache.tar ${{ variables.docker_image }}
+              displayName: Docker save
+              condition: and(not(canceled()), or(failed(), ne(variables.CACHE_RESTORED, 'true')))
           - bash: docker run -v $PWD:/project -i --name ${{ variables.docker_name }} -d ${{ variables.docker_image }} /bin/bash
             displayName: Start up Docker container
 
+        - ${{ if env['cache_dirs'] }}:
+          - ${{ each cache in env['cache_dirs'] }}:
+            - task: Cache@2
+              inputs: ${{ cache }}
+              displayName: Processing cache at ${{ cache['path'] }}
+        - ${{ elseif parameters.cache_dirs }}:
+          - ${{ each cache in parameters.cache_dirs }}:
+            - task: Cache@2
+              inputs: ${{ cache }}
+              displayName: Processing cache at ${{ cache['path'] }}
+
         - ${{ each tool_pair in coalesce(env['libraries'], parameters.libraries) }}:
           - ${{ each library in tool_pair.value }}:
 

diff --git a/test.py b/test.py
@@ -3,6 +3,7 @@
 
 import pytest
 
+
 def test_32bit():
     if '32' in os.environ['ENVNAME']:
         assert sys.maxsize == 2147483647
@@ -20,3 +21,21 @@ def test_opengl():
         assert len(version) > 0
     else:
          pytest.skip()
+
+
+@pytest.mark.skipif('cache-a' not in os.environ['ENVNAME'], reason='env not cache-a')
+@pytest.mark.parametrize('name,dir', [('data_x', 'cache_1'), ('data_y', 'cache_2')])
+def test_cache_a(name, dir):
+    assert not os.path.exists(dir)  # should be a cache miss
+    os.makedirs(dir)
+    with open(os.path.join(dir, 'test.txt'), 'w') as f:
+        f.write(f'writing to {name}:{dir} cache in cache-a')
+
+
+@pytest.mark.skipif('cache-b' not in os.environ['ENVNAME'], reason='env not cache-b')
+def test_cache_b():
+    assert not os.path.exists('cache_1')  # should be a cache miss
+    os.makedirs('cache_1')  # (to cache at end of job without failing)
+    assert not os.path.exists('cache_2')  # should not have loaded global cache
+    with open(os.path.join('cache_3', 'test.txt'), 'r') as f:  # should load from cache-a
+        assert f.readline() == f'writing to data_x:cache_1 cache in cache-a'
diff --git a/tox.ini b/tox.ini
@@ -9,6 +9,9 @@ envlist =
   opengl-windows
   compiler_macos_conda
   testtoxargs
+  cache-a
+  cache-b
+  cache-c
 
 requires = pip >= 18.0
            setuptools >= 30.3.0
@@ -20,7 +23,7 @@ deps =
     pytest
     opengl: vispy
     opengl: PyQt5
-commands = pytest test.py
+commands = pytest -v -s test.py
 
 [testenv:compiler_macos_conda]
 # This is a test to make sure that compilers work properly on MacOS X