Speed up GitHub workflows

.github/workflows/Publish_NIMS.yml

@@ -25,22 +25,20 @@ jobs:
     name: Update API reference docs and Publish NIMS Package to PyPI
     runs-on : ubuntu-latest 
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      - uses: Gr1N/setup-poetry@v8
+      - name: Set up Poetry
+        uses: Gr1N/setup-poetry@v8
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
 
       - name: Check for lock changes
         run: poetry lock --check
 
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
-
       # If the tag is 0.1.0, this will set the version of NIMS package to 0.1.0
       - name: Store version from Tag
         id: vars

.github/workflows/check_examples.yml

@@ -23,12 +23,30 @@ jobs:
         uses: Gr1N/setup-poetry@v8
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
-      - name: Cache Poetry virtualenv
+      # Updating poetry.lock for all of the examples takes over 6 minutes, so it's worth caching.
+      - name: Cache poetry.lock
         uses: actions/cache@v3
+        id: cache-poetry-lock
         with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
-      # Install each example's dependencies so mypy can see their types.
+          path: 'examples/**/poetry.lock'
+          # Include the main project's poetry.lock in the hash to detect upstream dependency updates.
+          key: examples-poetry-lock-${{ runner.os }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('examples/**/pyproject.toml', 'poetry.lock') }}
+      - name: Lock examples
+        if: steps.cache-poetry-lock.outputs.cache-hit != 'true'
+        run: |
+          for example in examples/*/; do
+            echo "::group::$example"
+            pushd $example
+            poetry lock
+            popd
+            echo "::endgroup::"
+          done
+      - name: Cache virtualenvs
+        uses: actions/cache@v3
+        id: cache-venv
+        with:
+          path: 'examples/**/.venv'
+          key: examples-venv-${{ runner.os }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('examples/**/poetry.lock') }}
       - name: Install examples
         run: |
           for example in examples/*/; do

.github/workflows/check_nimg.yml

@@ -29,11 +29,12 @@ jobs:
           poetry-version: ${{ env.POETRY_VERSION }}
       - name: Check for lock changes (ni-measurementlink-generator)
         run: poetry lock --check
-      - name: Cache Poetry virtualenv
+      - name: Cache virtualenv (ni-measurementlink-generator)
         uses: actions/cache@v3
+        id: cache
         with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          path: ni_measurementlink_generator/.venv
+          key: ni-measurementlink-generator-${{ runner.os }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('poetry.lock') }}
       - name: Install ni-measurementlink-generator
         run: poetry install -v
       - name: Lint ni-measurementlink-generator

.github/workflows/check_nims.yml

@@ -25,13 +25,22 @@ jobs:
           poetry-version: ${{ env.POETRY_VERSION }}
       - name: Check for lock changes (ni-measurementlink-service)
         run: poetry lock --check
-      - name: Cache Poetry virtualenv
-        uses: actions/cache@v3
+
+      # ni-measurementlink-service, all extras
+      - name: Restore cached virtualenv (ni-measurementlink-service, all extras)
+        uses: actions/cache/restore@v3
+        id: restore-nims-all-extras
         with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          path: .venv
+          key: ni-measurementlink-service-all-extras-${{ runner.os }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('poetry.lock') }}
       - name: Install ni-measurementlink-service (all extras)
         run: poetry install -v --all-extras
+      - name: Save cached virtualenv (ni-measurementlink-service, all extras)
+        uses: actions/cache/save@v3
+        if: steps.restore-nims-all-extras.outputs.cache-hit != 'true'
+        with:
+          path: .venv
+          key: ${{ steps.restore-nims-all-extras.outputs.cache-primary-key }}
       - name: Lint ni-measurementlink-service
         run: poetry run ni-python-styleguide lint
       - name: Mypy static analysis (ni-measurementlink-service, Linux)
@@ -42,12 +51,26 @@ jobs:
         run:  poetry run mypy tests
       - name: Mypy static analysis (tests, Windows)
         run:  poetry run mypy tests --platform win32
+
+      # ni-measurementlink-service, all extras, docs
+      - name: Restore cached virtualenv (ni-measurementlink-service, all extras, docs)
+        uses: actions/cache/restore@v3
+        id: restore-nims-all-extras-docs
+        with:
+          path: .venv
+          key: ni-measurementlink-service-all-extras-docs-${{ runner.os }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('poetry.lock') }}
       - name: Install ni-measurementlink-service (all extras, docs)
         run: poetry install -v --all-extras --with docs
+      - name: Save cached virtualenv (ni-measurementlink-service, all extras, docs)
+        uses: actions/cache/save@v3
+        if: steps.restore-nims-all-extras-docs.outputs.cache-hit != 'true'
+        with:
+          path: .venv
+          key: ${{ steps.restore-nims-all-extras-docs.outputs.cache-primary-key }}
       - name: Build docs and check for errors/warnings
         run: |
           rm -rf docs
           mkdir -p docs   
           poetry run sphinx-build _docs_source docs -b html -W --keep-going
       - name: Revert docs
-        run:  git clean -dfx docs/ && git restore docs/
+        run:  git clean -dfx docs/ && git restore docs/

.github/workflows/run_system_tests.yml

@@ -22,13 +22,21 @@ jobs:
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
+      - name: Cache virtualenvs
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: |
+            .venv
+            .tox
+          key: run-system-tests-${{ runner.os }}-${{ matrix.configuration }}-${{ hashFiles('poetry.lock') }}
       - name: Install dependencies
-        run: poetry install
+        run: poetry install -v
       - name: Run system tests
         run: poetry run tox
       - name: Upload test results
         uses: actions/upload-artifact@v3
         with:
           name: test_results
           path: test_results/*.xml
-        if: always()
+        if: always()

.github/workflows/run_unit_tests.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-latest]
+        os: [windows-latest, ubuntu-latest]
         python-version: [3.8, 3.9, '3.10', 3.11, 3.12]
       # Fail-fast skews the pass/fail ratio and seems to make pytest produce
       # incomplete JUnit XML results.
@@ -29,25 +29,63 @@ jobs:
         uses: Gr1N/setup-poetry@v8
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
-      - name: Cache Poetry virtualenv
-        uses: actions/cache@v3
+
+      # ni-measurementlink-service, no extras
+      - name: Restore cached virtualenv (ni-measurementlink-service, no extras)
+        uses: actions/cache/restore@v3
+        id: restore-nims-no-extras
         with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          path: .venv
+          key: ni-measurementlink-service-no-extras-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
       - name: Install ni-measurementlink-service (no extras)
         run: poetry install -v
+      - name: Save cached virtualenv (ni-measurementlink-service, no extras)
+        uses: actions/cache/save@v3
+        if: steps.restore-nims-no-extras.outputs.cache-hit != 'true'
+        with:
+          path: .venv
+          key: ${{ steps.restore-nims-no-extras.outputs.cache-primary-key }}
       - name: Run unit tests and code coverage (ni-measurementlink-service, no extras)
         run: poetry run pytest ./tests/unit -v --cov=ni_measurementlink_service --junitxml=test_results/nims-${{ matrix.os }}-py${{ matrix.python-version}}-no-extras.xml
+
+      # ni-measurementlink-service, all extras
+      - name: Restore cached virtualenv (ni-measurementlink-service, all extras)
+        uses: actions/cache/restore@v3
+        id: restore-nims-all-extras
+        with:
+          path: .venv
+          key: ni-measurementlink-service-all-extras-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
       - name: Install ni-measurementlink-service (all extras)
         run: poetry install -v --all-extras
+      - name: Save cached ni-measurementlink-service virtualenv (all extras)
+        uses: actions/cache/save@v3
+        if: steps.restore-nims-all-extras.outputs.cache-hit != 'true'
+        with:
+          path: .venv
+          key: ${{ steps.restore-nims-all-extras.outputs.cache-primary-key }}
       - name: Run unit tests and code coverage (ni-measurementlink-service, all extras)
         run: poetry run pytest ./tests/unit -v --cov=ni_measurementlink_service --junitxml=test_results/nims-${{ matrix.os }}-py${{ matrix.python-version}}-all-extras.xml
+
+      # ni-measurementlink-generator
+      - name: Restore cached virtualenv (ni-measurementlink-generator)
+        uses: actions/cache/restore@v3
+        id: restore-nimg
+        with:
+          path: ni_measurementlink_generator/.venv
+          key: ni-measurementlink-generator-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
       - name: Install ni-measurementlink-generator
         run: poetry install -v
         working-directory: ./ni_measurementlink_generator
+      - name: Save cached virtualenv (ni-measurementlink-generator)
+        uses: actions/cache/save@v3
+        if: steps.restore-nimg.outputs.cache-hit != 'true'
+        with:
+          path: ni_measurementlink_generator/.venv
+          key: ${{ steps.restore-nimg.outputs.cache-primary-key }}
       - name: Run tests and code coverage (ni-measurementlink-generator)
         run: poetry run pytest -v --cov=ni_measurementlink_generator --junitxml=test_results/nimg-${{ matrix.os }}-py${{ matrix.python-version}}.xml
         working-directory: ./ni_measurementlink_generator
+
       - name: Upload test results
         uses: actions/upload-artifact@v3
         with: