From 29aa84e5559e771e3b70ab7faae665e1c385ddc8 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 09:14:17 +0100 Subject: [PATCH 01/16] Progress towards no-codecov --- .github/workflows/ci.yml | 64 ++++++++++++++++++--------- .github/workflows/skip-tests.yml | 75 +++++++++++++++++++------------- .github/workflows/windows-ci.yml | 75 +++++++++++++++++++------------- README.rst | 4 +- codecov.yaml | 7 --- pyproject.toml | 1 - 6 files changed, 133 insertions(+), 93 deletions(-) delete mode 100644 codecov.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 676ccd6e4..60288e070 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,9 +120,6 @@ jobs: steps: - uses: actions/checkout@v5 - with: - # See https://github.com/codecov/codecov-action/issues/190. - fetch-depth: 2 - name: Install uv uses: astral-sh/setup-uv@v6 @@ -179,24 +176,13 @@ jobs: # https://github.com/VWS-Python/vws-python-mock/issues/708 cat ./coverage.xml - # We run this job on every PR, on every merge to main, and nightly. - # This causes us to hit an issue with Codecov. - # - # We see "Too many uploads to this commit.". - # See https://community.codecov.io/t/too-many-uploads-to-this-commit/2574. - # - # To work around this, we do not upload coverage data on scheduled runs. - # We print the event name here to help with debugging. - - name: Show event name - run: | - echo ${{ github.event_name }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + - name: Upload coverage data + uses: actions/upload-artifact@v4 with: - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + name: coverage-data-${{ matrix.python-version }}-${{ matrix.ci_pattern }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: ignore completion-ci: needs: build @@ -209,3 +195,41 @@ jobs: echo "One or more matrix jobs failed" exit 1 fi + + coverage: + name: Combine & check coverage + if: always() + needs: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + # Use latest Python, so it understands all syntax. + python-version: '3.13' + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-* + merge-multiple: true + + - name: Combine coverage & fail if it's <100% + run: | + uv tool install 'coverage[toml]' + + coverage combine + coverage html --skip-covered --skip-empty + + # Report and write to summary. + coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + + # Report again and fail if under 100%. + coverage report --fail-under=100 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 + with: + name: html-report + path: htmlcov + if: ${{ failure() }} diff --git a/.github/workflows/skip-tests.yml b/.github/workflows/skip-tests.yml index 2a3107c5d..8beeaa7f8 100644 --- a/.github/workflows/skip-tests.yml +++ b/.github/workflows/skip-tests.yml @@ -27,9 +27,6 @@ jobs: steps: - uses: actions/checkout@v5 - with: - # See https://github.com/codecov/codecov-action/issues/190. - fetch-depth: 2 - name: Install uv uses: astral-sh/setup-uv@v6 @@ -58,35 +55,13 @@ jobs: env: UV_PYTHON: ${{ matrix.python-version }} - - name: Show coverage file - run: | - # Sometimes we have been sure that we have 100% coverage, but codecov - # says otherwise. - # - # We show the coverage file here to help with debugging. - # https://github.com/VWS-Python/vws-python-mock/issues/708 - cat ./coverage.xml - - # We run this job on every PR, on every merge to main, and nightly. - # This causes us to hit an issue with Codecov. - # - # We see "Too many uploads to this commit.". - # See https://community.codecov.io/t/too-many-uploads-to-this-commit/2574. - # - # To work around this, we do not upload coverage data on scheduled runs. - # We print the event name here to help with debugging. - - name: Show event name - run: | - echo ${{ github.event_name }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + - name: Upload coverage data + uses: actions/upload-artifact@v4 with: - fail_ci_if_error: true - # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - # which tells us to use the token to avoid errors. - token: ${{ secrets.CODECOV_TOKEN }} - if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + name: coverage-data-skip-tests-${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: ignore completion-skip-tests: needs: build @@ -99,3 +74,41 @@ jobs: echo "One or more matrix jobs failed" exit 1 fi + + coverage-skip-tests: + name: Combine & check coverage (skip-tests) + if: always() + needs: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + # Use latest Python, so it understands all syntax. + python-version: '3.13' + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-skip-tests-* + merge-multiple: true + + - name: Combine coverage & fail if it's <100% + run: | + uv tool install 'coverage[toml]' + + coverage combine + coverage html --skip-covered --skip-empty + + # Report and write to summary. + coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + + # Report again and fail if under 100%. + coverage report --fail-under=100 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 + with: + name: html-report-skip-tests + path: htmlcov + if: ${{ failure() }} diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml index 3dd3ff0a2..fc1f580b5 100644 --- a/.github/workflows/windows-ci.yml +++ b/.github/workflows/windows-ci.yml @@ -25,9 +25,6 @@ jobs: steps: - uses: actions/checkout@v5 - with: - # See https://github.com/codecov/codecov-action/issues/190. - fetch-depth: 2 - name: Install uv uses: astral-sh/setup-uv@v6 @@ -47,35 +44,13 @@ jobs: env: UV_PYTHON: ${{ matrix.python-version }} - - name: Show coverage file - run: | - # Sometimes we have been sure that we have 100% coverage, but codecov - # says otherwise. - # - # We show the coverage file here to help with debugging. - # https://github.com/VWS-Python/vws-python-mock/issues/708 - cat ./coverage.xml - - # We run this job on every PR, on every merge to main, and nightly. - # This causes us to hit an issue with Codecov. - # - # We see "Too many uploads to this commit.". - # See https://community.codecov.io/t/too-many-uploads-to-this-commit/2574. - # - # To work around this, we do not upload coverage data on scheduled runs. - # We print the event name here to help with debugging. - - name: Show event name - run: | - echo ${{ github.event_name }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + - name: Upload coverage data + uses: actions/upload-artifact@v4 with: - fail_ci_if_error: true - # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - # which tells us to use the token to avoid errors. - token: ${{ secrets.CODECOV_TOKEN }} - if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + name: coverage-data-windows-${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: ignore completion-windows-ci: needs: build @@ -88,3 +63,41 @@ jobs: echo "One or more matrix jobs failed" exit 1 fi + + coverage-windows: + name: Combine & check coverage (Windows) + if: always() + needs: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + # Use latest Python, so it understands all syntax. + python-version: '3.13' + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-windows-* + merge-multiple: true + + - name: Combine coverage & fail if it's <100% + run: | + uv tool install 'coverage[toml]' + + coverage combine + coverage html --skip-covered --skip-empty + + # Report and write to summary. + coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + + # Report again and fail if under 100%. + coverage report --fail-under=100 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 + with: + name: html-report-windows + path: htmlcov + if: ${{ failure() }} diff --git a/README.rst b/README.rst index c2a566463..98004f17c 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -|Build Status| |codecov| |PyPI| +|Build Status| |PyPI| VWS Mock ======== @@ -56,8 +56,6 @@ This includes details on how to use the mock, options, and details of the differ .. |Build Status| image:: https://github.com/VWS-Python/vws-python-mock/actions/workflows/ci.yml/badge.svg?branch=main :target: https://github.com/VWS-Python/vws-python-mock/actions -.. |codecov| image:: https://codecov.io/gh/VWS-Python/vws-python-mock/branch/main/graph/badge.svg - :target: https://codecov.io/gh/VWS-Python/vws-python-mock .. |PyPI| image:: https://badge.fury.io/py/VWS-Python-Mock.svg :target: https://badge.fury.io/py/VWS-Python-Mock .. |minimum-python-version| replace:: 3.13 diff --git a/codecov.yaml b/codecov.yaml deleted file mode 100644 index 5c35baac9..000000000 --- a/codecov.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -coverage: - status: - patch: - default: - # Require 100% test coverage. - target: 100% diff --git a/pyproject.toml b/pyproject.toml index c2da135d8..627672bac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -313,7 +313,6 @@ ignore = [ "Makefile", "ci", "ci/**", - "codecov.yaml", "docs", "docs/**", ".git_archival.txt", From e3085791c860d49103cea3335c71b6ff37ba2872 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 09:17:50 +0100 Subject: [PATCH 02/16] Try to remove codecov --- .github/workflows/ci.yml | 50 ++------------------------------ .github/workflows/coverage.yml | 45 ++++++++++++++++++++++++++++ .github/workflows/skip-tests.yml | 38 ------------------------ .github/workflows/windows-ci.yml | 38 ------------------------ 4 files changed, 47 insertions(+), 124 deletions(-) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60288e070..1c36d20c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,19 +167,11 @@ jobs: env: UV_PYTHON: ${{ matrix.python-version }} - - name: Show coverage file - run: | - # Sometimes we have been sure that we have 100% coverage, but codecov - # says otherwise. - # - # We show the coverage file here to help with debugging. - # https://github.com/VWS-Python/vws-python-mock/issues/708 - cat ./coverage.xml - - name: Upload coverage data uses: actions/upload-artifact@v4 with: - name: coverage-data-${{ matrix.python-version }}-${{ matrix.ci_pattern }} + name: | + coverage-data-ci-${{ matrix.python-version }}-${{ matrix.ci_pattern}} path: .coverage.* include-hidden-files: true if-no-files-found: ignore @@ -195,41 +187,3 @@ jobs: echo "One or more matrix jobs failed" exit 1 fi - - coverage: - name: Combine & check coverage - if: always() - needs: build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - # Use latest Python, so it understands all syntax. - python-version: '3.13' - - - uses: actions/download-artifact@v4 - with: - pattern: coverage-data-* - merge-multiple: true - - - name: Combine coverage & fail if it's <100% - run: | - uv tool install 'coverage[toml]' - - coverage combine - coverage html --skip-covered --skip-empty - - # Report and write to summary. - coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" - - # Report again and fail if under 100%. - coverage report --fail-under=100 - - - name: Upload HTML report if check failed - uses: actions/upload-artifact@v4 - with: - name: html-report - path: htmlcov - if: ${{ failure() }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..aad465758 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,45 @@ +--- +name: Coverage + +on: + workflow_run: + workflows: [CI, Skip tests, Windows CI] + types: [completed] + +jobs: + coverage: + name: Combine & check coverage + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + # Use latest Python, so it understands all syntax. + python-version: '3.13' + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-* + merge-multiple: true + + - name: Combine coverage & fail if it's <100% + run: | + uv tool install 'coverage[toml]' + + coverage combine + coverage html --skip-covered --skip-empty + + # Report and write to summary. + coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + + # Report again and fail if under 100%. + coverage report --fail-under=100 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 + with: + name: html-report + path: htmlcov + if: ${{ failure() }} diff --git a/.github/workflows/skip-tests.yml b/.github/workflows/skip-tests.yml index 8beeaa7f8..fb9fea203 100644 --- a/.github/workflows/skip-tests.yml +++ b/.github/workflows/skip-tests.yml @@ -74,41 +74,3 @@ jobs: echo "One or more matrix jobs failed" exit 1 fi - - coverage-skip-tests: - name: Combine & check coverage (skip-tests) - if: always() - needs: build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - # Use latest Python, so it understands all syntax. - python-version: '3.13' - - - uses: actions/download-artifact@v4 - with: - pattern: coverage-data-skip-tests-* - merge-multiple: true - - - name: Combine coverage & fail if it's <100% - run: | - uv tool install 'coverage[toml]' - - coverage combine - coverage html --skip-covered --skip-empty - - # Report and write to summary. - coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" - - # Report again and fail if under 100%. - coverage report --fail-under=100 - - - name: Upload HTML report if check failed - uses: actions/upload-artifact@v4 - with: - name: html-report-skip-tests - path: htmlcov - if: ${{ failure() }} diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml index fc1f580b5..16bcaff76 100644 --- a/.github/workflows/windows-ci.yml +++ b/.github/workflows/windows-ci.yml @@ -63,41 +63,3 @@ jobs: echo "One or more matrix jobs failed" exit 1 fi - - coverage-windows: - name: Combine & check coverage (Windows) - if: always() - needs: build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - with: - # Use latest Python, so it understands all syntax. - python-version: '3.13' - - - uses: actions/download-artifact@v4 - with: - pattern: coverage-data-windows-* - merge-multiple: true - - - name: Combine coverage & fail if it's <100% - run: | - uv tool install 'coverage[toml]' - - coverage combine - coverage html --skip-covered --skip-empty - - # Report and write to summary. - coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" - - # Report again and fail if under 100%. - coverage report --fail-under=100 - - - name: Upload HTML report if check failed - uses: actions/upload-artifact@v4 - with: - name: html-report-windows - path: htmlcov - if: ${{ failure() }} From b24da0fb1c99ce4314264870fb338e18da6f9805 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 09:34:57 +0100 Subject: [PATCH 03/16] Try using setup-uv --- .github/workflows/coverage.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index aad465758..a8f80c7dd 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -14,10 +14,11 @@ jobs: steps: - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v6 with: - # Use latest Python, so it understands all syntax. - python-version: '3.13' + enable-cache: true + cache-dependency-glob: '**/pyproject.toml' - uses: actions/download-artifact@v4 with: From f4d0840f4db63598c50c1ccd2be417dc4110f529 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 09:56:24 +0100 Subject: [PATCH 04/16] Try to fix coverage workflow to run --- .github/workflows/coverage.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a8f80c7dd..1c9d22089 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -2,18 +2,36 @@ name: Coverage on: - workflow_run: - workflows: [CI, Skip tests, Windows CI] - types: [completed] + push: + branches: [main] + pull_request: + branches: [main] + schedule: + # * is a special character in YAML so you have to quote this string + # Run at 1:00 every day + - cron: 0 1 * * * + workflow_dispatch: {} jobs: coverage: name: Combine & check coverage - if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + + - name: Wait for other workflows + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ github.ref }} + check-name: | + CI + Skip tests + Windows CI + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 10 + allowed-conclusions: success + - name: Install uv uses: astral-sh/setup-uv@v6 with: From 304f484b3e15b5f2e2f9fe525c4c5e363833b483 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 10:09:39 +0100 Subject: [PATCH 05/16] Try combining test jobs --- .github/workflows/coverage.yml | 64 ---------- .github/workflows/skip-tests.yml | 76 ----------- .github/workflows/{ci.yml => test.yml} | 168 ++++++++++++++++++++++--- .github/workflows/windows-ci.yml | 65 ---------- 4 files changed, 150 insertions(+), 223 deletions(-) delete mode 100644 .github/workflows/coverage.yml delete mode 100644 .github/workflows/skip-tests.yml rename .github/workflows/{ci.yml => test.yml} (66%) delete mode 100644 .github/workflows/windows-ci.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 1c9d22089..000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -name: Coverage - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - # * is a special character in YAML so you have to quote this string - # Run at 1:00 every day - - cron: 0 1 * * * - workflow_dispatch: {} - -jobs: - coverage: - name: Combine & check coverage - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - - name: Wait for other workflows - uses: lewagon/wait-on-check-action@v1.3.4 - with: - ref: ${{ github.ref }} - check-name: | - CI - Skip tests - Windows CI - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 10 - allowed-conclusions: success - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - cache-dependency-glob: '**/pyproject.toml' - - - uses: actions/download-artifact@v4 - with: - pattern: coverage-data-* - merge-multiple: true - - - name: Combine coverage & fail if it's <100% - run: | - uv tool install 'coverage[toml]' - - coverage combine - coverage html --skip-covered --skip-empty - - # Report and write to summary. - coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" - - # Report again and fail if under 100%. - coverage report --fail-under=100 - - - name: Upload HTML report if check failed - uses: actions/upload-artifact@v4 - with: - name: html-report - path: htmlcov - if: ${{ failure() }} diff --git a/.github/workflows/skip-tests.yml b/.github/workflows/skip-tests.yml deleted file mode 100644 index fb9fea203..000000000 --- a/.github/workflows/skip-tests.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- - -# We check that using all --skip options does not error. - -name: Skip tests - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - # * is a special character in YAML so you have to quote this string - # Run at 1:00 every day - - cron: 0 1 * * * - workflow_dispatch: {} - -jobs: - build: - - strategy: - matrix: - python-version: ['3.13'] - platform: [ubuntu-latest] - - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/checkout@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - cache-dependency-glob: '**/pyproject.toml' - - - name: Set secrets file - run: | - cp ./vuforia_secrets.env.example ./vuforia_secrets.env - - - name: Run tests - run: | - uv run --extra=dev pytest \ - --skip-docker_build_tests \ - --skip-docker_in_memory \ - --skip-mock \ - --skip-real \ - --capture=no \ - -vvv \ - --exitfirst \ - --cov=src/ \ - --cov=tests/ \ - --cov-report=xml \ - . - env: - UV_PYTHON: ${{ matrix.python-version }} - - - name: Upload coverage data - uses: actions/upload-artifact@v4 - with: - name: coverage-data-skip-tests-${{ matrix.python-version }} - path: .coverage.* - include-hidden-files: true - if-no-files-found: ignore - - completion-skip-tests: - needs: build - runs-on: ubuntu-latest - if: always() # Run even if one matrix job fails - steps: - - name: Check matrix job status - run: |- - if ! ${{ needs.build.result == 'success' }}; then - echo "One or more matrix jobs failed" - exit 1 - fi diff --git a/.github/workflows/ci.yml b/.github/workflows/test.yml similarity index 66% rename from .github/workflows/ci.yml rename to .github/workflows/test.yml index 1c36d20c2..c74fd506a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,5 @@ --- - -name: CI +name: Test on: push: @@ -13,16 +12,13 @@ on: - cron: 0 1 * * * workflow_dispatch: {} -# We share Vuforia credentials and therefore Vuforia databases across -# workflows. -# We therefore want to run only one workflow at a time. -concurrency: vuforia_credentials - jobs: - build: - + # CI tests with matrix + ci-tests: runs-on: ubuntu-latest - + # We share Vuforia credentials and therefore Vuforia databases across + # workflows. We therefore want to run only one workflow at a time. + concurrency: vuforia_credentials strategy: fail-fast: false matrix: @@ -170,20 +166,156 @@ jobs: - name: Upload coverage data uses: actions/upload-artifact@v4 with: - name: | - coverage-data-ci-${{ matrix.python-version }}-${{ matrix.ci_pattern}} + name: coverage-data-ci-${{ matrix.python-version }}-${{ matrix.ci_pattern + }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: ignore + + # Skip tests + skip-tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.13'] + platform: [ubuntu-latest] + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: '**/pyproject.toml' + + - name: Set secrets file + run: | + cp ./vuforia_secrets.env.example ./vuforia_secrets.env + + - name: Run tests + run: | + uv run --extra=dev pytest \ + --skip-docker_build_tests \ + --skip-docker_in_memory \ + --skip-mock \ + --skip-real \ + --capture=no \ + -vvv \ + --exitfirst \ + --cov=src/ \ + --cov=tests/ \ + --cov-report=xml \ + . + env: + UV_PYTHON: ${{ matrix.python-version }} + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: coverage-data-skip-tests-${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: ignore + + # Windows tests + windows-tests: + runs-on: windows-latest + strategy: + matrix: + python-version: ['3.13'] + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: '**/pyproject.toml' + + - name: Set secrets file + run: | + cp ./vuforia_secrets.env.example ./vuforia_secrets.env + + - name: Run tests + run: | + # We use pytest-xdist to make this run much faster. + # The downside is that we cannot use -s / --capture=no. + uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov=src/ --cov=tests/ --cov-report=xml . + env: + UV_PYTHON: ${{ matrix.python-version }} + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: coverage-data-windows-${{ matrix.python-version }} path: .coverage.* include-hidden-files: true if-no-files-found: ignore - completion-ci: - needs: build + # Coverage combination and enforcement + coverage: + name: Combine & check coverage + needs: [ci-tests, skip-tests, windows-tests] + if: always() runs-on: ubuntu-latest - if: always() # Run even if one matrix job fails + steps: - - name: Check matrix job status + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: '**/pyproject.toml' + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-* + merge-multiple: true + + - name: Combine coverage & fail if it's <100% + run: | + uv tool install 'coverage[toml]' + + coverage combine + coverage html --skip-covered --skip-empty + + # Report and write to summary. + coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + + # Report again and fail if under 100%. + coverage report --fail-under=100 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 + with: + name: html-report + path: htmlcov + if: ${{ failure() }} + + # Final completion check + completion: + needs: [ci-tests, skip-tests, windows-tests, coverage] + runs-on: ubuntu-latest + if: always() + steps: + - name: Check all jobs status run: |- - if ! ${{ needs.build.result == 'success' }}; then - echo "One or more matrix jobs failed" + if ! ${{ needs.ci-tests.result == 'success' }}; then + echo "CI tests failed" + exit 1 + fi + if ! ${{ needs.skip-tests.result == 'success' }}; then + echo "Skip tests failed" + exit 1 + fi + if ! ${{ needs.windows-tests.result == 'success' }}; then + echo "Windows tests failed" + exit 1 + fi + if ! ${{ needs.coverage.result == 'success' }}; then + echo "Coverage check failed" exit 1 fi diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml deleted file mode 100644 index 16bcaff76..000000000 --- a/.github/workflows/windows-ci.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- - -name: Windows CI - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - # * is a special character in YAML so you have to quote this string - # Run at 1:00 every day - - cron: 0 1 * * * - workflow_dispatch: {} - -jobs: - build: - - strategy: - matrix: - python-version: ['3.13'] - platform: [windows-latest] - - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/checkout@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - cache-dependency-glob: '**/pyproject.toml' - - - name: Set secrets file - run: | - cp ./vuforia_secrets.env.example ./vuforia_secrets.env - - - name: Run tests - run: | - # We use pytest-xdist to make this run much faster. - # The downside is that we cannot use -s / --capture=no. - uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov=src/ --cov=tests/ --cov-report=xml . - env: - UV_PYTHON: ${{ matrix.python-version }} - - - name: Upload coverage data - uses: actions/upload-artifact@v4 - with: - name: coverage-data-windows-${{ matrix.python-version }} - path: .coverage.* - include-hidden-files: true - if-no-files-found: ignore - - completion-windows-ci: - needs: build - runs-on: ubuntu-latest - if: always() # Run even if one matrix job fails - steps: - - name: Check matrix job status - run: |- - if ! ${{ needs.build.result == 'success' }}; then - echo "One or more matrix jobs failed" - exit 1 - fi From de06ee0514173be0620d3201bc3a88aa93db1b4f Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 10:15:44 +0100 Subject: [PATCH 06/16] Update custom linter --- ci/test_custom_linters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index 8de79cbb8..f740f39a0 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -18,9 +18,9 @@ def _ci_patterns(*, repository_root: Path) -> set[str]: """ Return the CI patterns given in the CI configuration file. """ - ci_file = repository_root / ".github" / "workflows" / "ci.yml" + ci_file = repository_root / ".github" / "workflows" / "test.yml" github_workflow_config = yaml.safe_load(stream=ci_file.read_text()) - matrix = github_workflow_config["jobs"]["build"]["strategy"]["matrix"] + matrix = github_workflow_config["jobs"]["ci-tests"]["strategy"]["matrix"] ci_pattern_list = matrix["ci_pattern"] ci_patterns = set(ci_pattern_list) assert len(ci_pattern_list) == len(ci_patterns) From 1834228d643171c8b2b21e325ccd0c3ae6e97d1d Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 10:36:00 +0100 Subject: [PATCH 07/16] Allow vuforia jobs to run in parallel --- .github/workflows/test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c74fd506a..d9982e8e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,13 +12,14 @@ on: - cron: 0 1 * * * workflow_dispatch: {} +# We share Vuforia credentials and therefore Vuforia databases across +# workflows. We therefore want to run only one workflow at a time. +concurrency: vuforia_credentials + jobs: # CI tests with matrix ci-tests: runs-on: ubuntu-latest - # We share Vuforia credentials and therefore Vuforia databases across - # workflows. We therefore want to run only one workflow at a time. - concurrency: vuforia_credentials strategy: fail-fast: false matrix: From 6ec29b01d86ca2a69ed42dc98b9a2794114ddba1 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:03:07 +0100 Subject: [PATCH 08/16] Error earlier if no fiels are found --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9982e8e1..71933870d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -171,7 +171,7 @@ jobs: }} path: .coverage.* include-hidden-files: true - if-no-files-found: ignore + if-no-files-found: error # Skip tests skip-tests: @@ -217,7 +217,7 @@ jobs: name: coverage-data-skip-tests-${{ matrix.python-version }} path: .coverage.* include-hidden-files: true - if-no-files-found: ignore + if-no-files-found: error # Windows tests windows-tests: @@ -253,7 +253,7 @@ jobs: name: coverage-data-windows-${{ matrix.python-version }} path: .coverage.* include-hidden-files: true - if-no-files-found: ignore + if-no-files-found: error # Coverage combination and enforcement coverage: From 646510d5c1bb865f2268020da93ed2b346c8a2f5 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:03:46 +0100 Subject: [PATCH 09/16] Add an ID to require 100% coverage step --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71933870d..7a0409a88 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -276,7 +276,8 @@ jobs: pattern: coverage-data-* merge-multiple: true - - name: Combine coverage & fail if it's <100% + - name: Require 100% Coverage + id: coverage run: | uv tool install 'coverage[toml]' From 3201034ecdf30abfa5ee40423b51b211aa7f1a52 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:05:31 +0100 Subject: [PATCH 10/16] Try --cov-append --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a0409a88..0663b7cb9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -157,6 +157,7 @@ jobs: -vvv \ --showlocals \ --exitfirst \ + --cov-append \ --cov=src/ \ --cov=tests/ \ --cov-report=xml \ @@ -204,6 +205,7 @@ jobs: --capture=no \ -vvv \ --exitfirst \ + --cov-append \ --cov=src/ \ --cov=tests/ \ --cov-report=xml \ @@ -243,7 +245,7 @@ jobs: run: | # We use pytest-xdist to make this run much faster. # The downside is that we cannot use -s / --capture=no. - uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov=src/ --cov=tests/ --cov-report=xml . + uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov-append --cov=src/ --cov=tests/ --cov-report=xml . env: UV_PYTHON: ${{ matrix.python-version }} From b8b4e412388a788e2c8e89bd7cdddf5cddfbf619 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:14:12 +0100 Subject: [PATCH 11/16] Do not write unnecessary XML --- .github/workflows/test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0663b7cb9..0e2146f35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -160,7 +160,6 @@ jobs: --cov-append \ --cov=src/ \ --cov=tests/ \ - --cov-report=xml \ ${{ matrix.ci_pattern }} env: UV_PYTHON: ${{ matrix.python-version }} @@ -208,7 +207,6 @@ jobs: --cov-append \ --cov=src/ \ --cov=tests/ \ - --cov-report=xml \ . env: UV_PYTHON: ${{ matrix.python-version }} @@ -245,7 +243,7 @@ jobs: run: | # We use pytest-xdist to make this run much faster. # The downside is that we cannot use -s / --capture=no. - uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov-append --cov=src/ --cov=tests/ --cov-report=xml . + uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov-append --cov=src/ --cov=tests/ . env: UV_PYTHON: ${{ matrix.python-version }} From 8fa90b53f451141abc89c8da095bfe395e6c6da7 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:24:43 +0100 Subject: [PATCH 12/16] Try without pytest-cov --- .github/workflows/test.yml | 62 +++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e2146f35..6da17f205 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -152,23 +152,25 @@ jobs: - name: Run tests run: | - uv run --extra=dev pytest \ - -s \ - -vvv \ - --showlocals \ - --exitfirst \ - --cov-append \ - --cov=src/ \ - --cov=tests/ \ - ${{ matrix.ci_pattern }} + uv run --extra=dev \ + coverage run \ + --parallel-mode \ + --source=src/ \ + --source=tests/ \ + -m pytest \ + -s \ + -vvv \ + --showlocals \ + --exitfirst \ + ${{ matrix.ci_pattern }} env: UV_PYTHON: ${{ matrix.python-version }} - name: Upload coverage data uses: actions/upload-artifact@v4 with: - name: coverage-data-ci-${{ matrix.python-version }}-${{ matrix.ci_pattern - }} + name: | + coverage-data-ci-${{ matrix.python-version }}-${{ matrix.ci_pattern}} path: .coverage.* include-hidden-files: true if-no-files-found: error @@ -196,18 +198,20 @@ jobs: - name: Run tests run: | - uv run --extra=dev pytest \ - --skip-docker_build_tests \ - --skip-docker_in_memory \ - --skip-mock \ - --skip-real \ - --capture=no \ - -vvv \ - --exitfirst \ - --cov-append \ - --cov=src/ \ - --cov=tests/ \ - . + uv run --extra=dev \ + coverage run \ + --parallel-mode \ + --source=src/ \ + --source=tests/ \ + -m pytest \ + --skip-docker_build_tests \ + --skip-docker_in_memory \ + --skip-mock \ + --skip-real \ + --capture=no \ + -vvv \ + --exitfirst \ + . env: UV_PYTHON: ${{ matrix.python-version }} @@ -243,7 +247,17 @@ jobs: run: | # We use pytest-xdist to make this run much faster. # The downside is that we cannot use -s / --capture=no. - uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov-append --cov=src/ --cov=tests/ . + uv run --extra=dev \ + coverage run \ + --parallel-mode \ + --source=src/ \ + --source=tests/ \ + -m pytest \ + --skip-real \ + -vvv \ + --exitfirst \ + -n auto \ + . env: UV_PYTHON: ${{ matrix.python-version }} From 58d25ceb2ea5032205e408fcc02b3e3b00305592 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:27:10 +0100 Subject: [PATCH 13/16] Switch to bash shell on Windows --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6da17f205..82d112cee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -244,6 +244,7 @@ jobs: cp ./vuforia_secrets.env.example ./vuforia_secrets.env - name: Run tests + shell: bash run: | # We use pytest-xdist to make this run much faster. # The downside is that we cannot use -s / --capture=no. From 5670e38e030a97e6ffa916dc3705478311f45dca Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:30:37 +0100 Subject: [PATCH 14/16] Sanitize pattern names --- .github/workflows/test.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82d112cee..5e966b674 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,11 +166,16 @@ jobs: env: UV_PYTHON: ${{ matrix.python-version }} + - name: Sanitize pattern for artifact name + id: sanitize + run: | + SANITIZED_PATTERN=$(echo "${{ matrix.ci_pattern }}" | sed 's/::/-/g' | sed 's/:/-/g' | sed 's|/|-|g') + echo "name=coverage-data-ci-${{ matrix.python-version }}-${SANITIZED_PATTERN}" >> "$GITHUB_OUTPUT" + - name: Upload coverage data uses: actions/upload-artifact@v4 with: - name: | - coverage-data-ci-${{ matrix.python-version }}-${{ matrix.ci_pattern}} + name: ${{ steps.sanitize.outputs.name }} path: .coverage.* include-hidden-files: true if-no-files-found: error From 9e568f60ce45a871960d24dc55f0e01e2a4e94f6 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 11:44:09 +0100 Subject: [PATCH 15/16] Try not doing Windows coverage --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e966b674..526172e0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -267,13 +267,13 @@ jobs: env: UV_PYTHON: ${{ matrix.python-version }} - - name: Upload coverage data - uses: actions/upload-artifact@v4 - with: - name: coverage-data-windows-${{ matrix.python-version }} - path: .coverage.* - include-hidden-files: true - if-no-files-found: error + # - name: Upload coverage data + # uses: actions/upload-artifact@v4 + # with: + # name: coverage-data-windows-${{ matrix.python-version }} + # path: .coverage.* + # include-hidden-files: true + # if-no-files-found: error # Coverage combination and enforcement coverage: From 957340153f58befa11a8ef69eb0831144c138ecb Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 7 Oct 2025 12:04:25 +0100 Subject: [PATCH 16/16] Tidy up a bit --- .github/workflows/test.yml | 15 ++++++--------- pyproject.toml | 2 +- tests/mock_vws/test_docker.py | 12 +++++++----- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 526172e0e..1042c73d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -253,6 +253,12 @@ jobs: run: | # We use pytest-xdist to make this run much faster. # The downside is that we cannot use -s / --capture=no. + # + # We use coverage to collect coverage data but we currently + # do not upload / use it because combining Windows and Linux + # coverage is challenging. + # + # We therefore have a few ``# pragma: no cover`` statements. uv run --extra=dev \ coverage run \ --parallel-mode \ @@ -267,15 +273,6 @@ jobs: env: UV_PYTHON: ${{ matrix.python-version }} - # - name: Upload coverage data - # uses: actions/upload-artifact@v4 - # with: - # name: coverage-data-windows-${{ matrix.python-version }} - # path: .coverage.* - # include-hidden-files: true - # if-no-files-found: error - - # Coverage combination and enforcement coverage: name: Combine & check coverage needs: [ci-tests, skip-tests, windows-tests] diff --git a/pyproject.toml b/pyproject.toml index 627672bac..051535a80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ optional-dependencies.dev = [ "actionlint-py==1.7.7.23", "check-manifest==0.50", "check-wheel-contents==0.6.3", + "coverage==7.10.7", "deptry==0.23.1", "dirty-equals==0.10.0", "doc8==1.1.1", @@ -72,7 +73,6 @@ optional-dependencies.dev = [ "pyright==1.1.406", "pyroma==5.0", "pytest==8.4.2", - "pytest-cov==7.0.0", "pytest-retry==1.7.0", "pytest-xdist==3.8.0", "python-dotenv==1.1.1", diff --git a/tests/mock_vws/test_docker.py b/tests/mock_vws/test_docker.py index 56c5fc6ce..d3d18c16a 100644 --- a/tests/mock_vws/test_docker.py +++ b/tests/mock_vws/test_docker.py @@ -65,7 +65,9 @@ def fixture_custom_bridge_network() -> Iterator[Network]: name = "test-vws-bridge-" + uuid.uuid4().hex try: network = client.networks.create(name=name, driver="bridge") - except NotFound: + # We skip coverage here because combining Windows and Linux coverage + # is challenging. + except NotFound: # pragma: no cover # On Windows the "bridge" network driver is not available and we use # the "nat" driver instead. network = client.networks.create(name=name, driver="nat") @@ -116,15 +118,15 @@ def test_build_and_run( target="target-manager", rm=True, ) - except BuildError as exc: + # We skip coverage here because combining Windows and Linux coverage + # is challenging. + except BuildError as exc: # pragma: no cover full_log = "\n".join( [item["stream"] for item in exc.build_log if "stream" in item], ) # If this assertion fails, it may be useful to look at the other # properties of ``exc``. - if ( - "no matching manifest for windows/amd64" not in exc.msg - ): # pragma: no cover + if "no matching manifest for windows/amd64" not in exc.msg: raise AssertionError(full_log) from exc pytest.skip( reason="We do not currently support using Windows containers."