diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000..d6a11e91 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,111 @@ +# For more configuration details: +# https://docs.codecov.io/docs/codecov-yaml + +# After making edits, check if this file is valid by running: +# curl -X POST --data-binary @.codecov.yml https://codecov.io/validate + +# +# Coverage configuration +# ---------------------- +# +github_checks: + # + # On adding coverage annotations to the code in the GitHub + # Code Review for now: + # + # - The annotations consume a lot of space in the PR code review, + # and can make it hard to review files that are not covered yet. + # + # - The coverage can be visited using the Codecov link at all times. + # https://app.codecov.io/gh/xapi-project/xen-api/pulls + # + # - The annotations can be hidden in GitHub PR code review by + # pressing the "a" key or by deselecting the "Show comments" + # checkbox but they are shown by default. + # + # - The Codecov Chrome and Firefox extension is a much nicer + # way to indicate coverage: + # + # Link: https://github.com/codecov/codecov-browser-extension + # + # - How to enable: You need to log in to Codecov using Github. + # For Firefox, enable the needed permissions: + # https://github.com/codecov/codecov-browser-extension/issues/50 + # + # Reference: + # http://docs.codecov.com/docs/common-recipe-list#disable-github-check-run-annotations + # + annotations: true + +codecov: + token: 37ee4f08-37b2-404b-bf4d-254891cf6f30 # Codecov coverage upload token + +# +# Pull request comments: +# ---------------------- +# This feature adds the code coverage summary as a comment on each PR. +# See https://docs.codecov.io/docs/pull-request-comments +# This same information is available from the Codecov checks in the PR's +# "Checks" tab in GitHub even when this feature is disabled. +# +comment: + # + # Legend: + # "diff" is the Coverage Diff of the pull request. + # "files" are the files impacted by the pull request + # "flags" are the coverage status of the pull request + # + # For an even shorter layout, this may be used: + # layout: "condensed_header, diff, files, flags" + # + layout: "header, diff, files, flags" + + # + # Add the Codecov comment to the PR when coverage changes always for testing + # + require_changes: false + + # + # The overall project coverage is secondary to the individual coverage + # and it is always shown in the repository at: + # - https://app.codecov.io/gh/xenserver/python-libs + # + hide_project_coverage: true + +coverage: + # + # Number of precision digits when showing coverage percentage e.g. 88.8%. + # One precision digit is also used by coverage.py when reporting coverage: + # + precision: 1 + + status: + + # + # Patch coverage is the incremental change in coverage in a PR + # + patch: + default: false # disable the default status that measures entire project + + tests: + paths: ["tests/"] # only include coverage in "tests/" folder + target: auto # don't reduce coverage on test code lines + + python-libs: # declare a new status context "python-libs" + paths: ["xcp/"] # library code + target: 0 # Temporarily allow 0% coverage to allow to merge dmv.py, + # Project threshold sets a lower bound to not go further. + + # + # Project coverage is the absolute coverage of the entire project + # + project: + default: false # disable the default status that measures entire project + + tests: # declare a new status context "tests" + paths: ["tests/"] # only include coverage in "tests/" folder + target: 99% # we always want 99% coverage here + + python-libs: # declare a new status context "python-libs" + paths: ["xcp/"] # library code + target: 78% # Coverage should not be reduced compared to its base diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 476a0293..5eddd4c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,6 +13,11 @@ concurrency: # On new workflow, cancel old workflows from the same PR, branch o # or requested by adding "request-checks: true" if disabled by default for pushes: # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks#skipping-and-requesting-checks-for-individual-commits on: [push, pull_request] + +permissions: + contents: read + pull-requests: write + env: PYTHONWARNINGS: "ignore:DEPRECATION" PIP_ROOT_USER_ACTION: "ignore" # For local testing using act-cli @@ -35,6 +40,8 @@ jobs: - python-version: '3.13' os: ubuntu-22.04 runs-on: ${{ matrix.os }} + env: + PYTHON_VERSION_USED_FOR_COVERAGE: ${{ '3.11' }} steps: - uses: actions/checkout@v4 with: @@ -50,94 +57,38 @@ jobs: if: ${{ (github.actor == 'nektos/act' && matrix.python-version != '3.11') }} run: apt-get update && apt-get install -y cpio - - name: Run of tox on Ubuntu + - name: Run tox to run pytest in the defined tox environments # Python 3.11 is not supported in the nektos/act container, so we skip this step if: ${{ !(matrix.python-version == '3.11' && github.actor == 'nektos/act') }} run: | pip install tox tox-gh-actions tox --workdir .github/workflows/.tox --recreate - - - name: Select the coverage file for upload - if: | - matrix.python-version == '3.11' && - ( !cancelled() && github.actor != 'nektos/act' ) - id: coverage - run: mv $( ls -t .github/workflows/.tox/*/log/.coverage | head -1 ) .coverage - - # The new reliable Codecov upload requires Codecov to query the GitHub API to check - # the repo and the commit. The repo (or organisation) owner needs to login to - # codecov, generated the CODECOV_TOKEN and save it as a secret in the ORG or the repo: - # https://docs.codecov.com/docs/adding-the-codecov-token - - # Links to get and set the token: - # Get the CODECOV_TOKEN: https://app.codecov.io/gh/xenserver/python-libs/settings - # Set the CODE_COV_TOKEN: https://github.com/xenserver/python-libs/settings/secrets/actions - - # Without it, the API calls are rate-limited by GitHub, and the upload may fail: - # https://github.com/codecov/feedback/issues/126#issuecomment-1932658904 - # - - name: Upload coverage reports to Codecov (fallback, legacy Node.js 16 action) - # If CODECOV_TOKEN is not set, use the legacy tokenless Codecov action: env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - # To reduce chances of GitHub's API throttling to hit this upload, only run the - # upload for the py38-covcombine-check job running on Ubuntu-20.04, which is the - # one we need. And only run it for PRs and the master branch, not for pushes. - # This reduces the number of uploads and the chance of hitting the rate limit - # by a factor of 6. + DIFF_COVERAGE_MIN: 0 # Let the reviewdog and codecov show uncovered lines + + - uses: aki77/reviewdog-action-code-coverage@v2 + continue-on-error: true if: | - steps.coverage.outcome == 'success' && - !env.CODECOV_TOKEN && !cancelled() && - matrix.os == 'ubuntu-20.04' && github.actor != 'nektos/act' && - ( github.event.pull_request.number || github.ref == 'refs/heads/master' ) - uses: codecov/codecov-action@v3 + matrix.python-version == env.PYTHON_VERSION_USED_FOR_COVERAGE && + !cancelled() && github.event_name == 'pull_request' with: - directory: .github/workflows/.tox/py38-covcombine-check/log - env_vars: OS,PYTHON - # Use fail_ci_if_error: false as explained the big comment above: - # Not failing this job in this case is ok because the tox CI checks also contain - # a diff-cover check which would fail on changed lines missing coverage. - # The Codecov CLI is more reliable and should be used if the CODECOV_TOKEN is set. - # The Codecov CLI is used in the next step when CODECOV_TOKEN is set. - fail_ci_if_error: false - flags: unittest - name: py27-py38-combined - verbose: true + lcov_path: coverage.lcov - - name: Upload coverage reports to Codecov (used when secrets.CODECOV_TOKEN is set) - # If CODECOV_TOKEN is set, use the new Codecov CLI to upload the coverage reports - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - uses: codecov/codecov-action@v5 + id: codecov if: | - env.CODECOV_TOKEN && !cancelled() && github.actor != 'nektos/act' && - steps.coverage.outcome == 'success' && matrix.os == 'ubuntu-20.04' - run: > - set -euxv; - mv .github/workflows/.tox/py38-covcombine-check/log/coverage.xml cov.xml; - curl -O https://cli.codecov.io/latest/linux/codecov; sudo chmod +x codecov; - ./codecov upload-process --report-type coverage - --name "CLI Upload for ${{ env.PYTHON_VERSION }}" - --git-service github --fail-on-error --file cov.xml --disable-search - --flag python${{ env.PYTHON_VERSION }} - continue-on-error: false # Fail the job if the upload with CODECOV_TOKEN fails + matrix.python-version == env.PYTHON_VERSION_USED_FOR_COVERAGE && + !cancelled() && github.actor != 'nektos/act' + + - uses: codecov/test-results-action@v1 + if: steps.codecov.outcome == 'success' - - if: steps.coverage.outcome == 'success' + - if: | + matrix.python-version == env.PYTHON_VERSION_USED_FOR_COVERAGE && + !cancelled() && github.actor != 'nektos/act' name: Upload coverage reports to Coveralls env: - COVERALLS_PARALLEL: true COVERALLS_FLAG_NAME: ${{ format('python{0}', steps.python.outputs.python-version ) }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: pip install coveralls && coveralls --service=github - - # For combined coverage of 2.7, 3.8 and 3.11 we upload to Coveralls in parallel mode. - # To view the Coveralls results from the PR, click on the "Details" link to the right - # of the Coveralls Logo in the Checks section of the PR. - finish-coverage-upload: - if: github.actor != 'nektos/act' - needs: test - runs-on: ubuntu-latest - steps: - - name: Finish the coverage upload to Coveralls - uses: coverallsapp/github-action@v1 - with: - parallel-finished: true + continue-on-error: true # Coveralls.io is currently overloaded diff --git a/tox.ini b/tox.ini index 535d80ec..84fb61b8 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,9 @@ commands = # https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md # https://github.com/actions/toolkit/blob/main/docs/commands.md#problem-matchers echo "::add-matcher::.github/workflows/PYTHONWARNINGS-problemMatcher.json" - pytest --cov -v --new-first -x --show-capture=all -rA + echo "::group::pytest" + pytest --cov -v --new-first -x --show-capture=all -rA --junitxml={envlogdir}/junit.xml -o junit_family=legacy + echo "::endgroup::" sh -c 'if [ -n "{env:PYTEST_MD_REPORT_OUTPUT}" -a -n "{env:GITHUB_STEP_SUMMARY}" ];then \ mkdir -p $(dirname "{env:GITHUB_STEP_SUMMARY:.git/sum.md}"); \ sed "s/tests\(.*py\)/[&](&)/" \ @@ -81,6 +83,9 @@ passenv = pytype: GITHUB_REF_NAME test: PYTEST_ADDOPTS test: PYTEST_XDIST_WORKER_COUNT + cov: TESTS_COVERAGE_MIN + cov: XCP_COVERAGE_MIN + cov: DIFF_COVERAGE_MIN covcp: UPLOAD_DIR covcp: HOME check: MYPY_FORCE_COLOR @@ -114,7 +119,7 @@ commands = # covcombine shall not call [cov]commands: diff-cover shall check the combined cov: {cov,covcp}: {[cov]commands} {py27-test}: pylint --py3k --disable=no-absolute-import xcp/ - covcp: cp -av {envlogdir}/coverage.xml {env:UPLOAD_DIR:.} + covcp: cp -av {envlogdir}/coverage.xml {envlogdir}/coverage.lcov {envlogdir}/.coverage {envlogdir}/junit.xml {env:UPLOAD_DIR:.} covcombine: {[covcombine]commands} fox: {[covcombine]commands} fox: {[lint]commands} @@ -127,12 +132,13 @@ setenv = PY3_DIFFCOVER_OPTIONS=--ignore-whitespace --show-uncovered extras = coverage test commands = - coverage xml -o {envlogdir}/coverage.xml --fail-under {env:XCP_COV_MIN:68} + coverage xml -o {envlogdir}/coverage.xml --fail-under {env:XCP_COVERAGE_MIN:78} + coverage lcov -o {envlogdir}/coverage.lcov coverage html -d {envlogdir}/htmlcov - coverage html -d {envlogdir}/htmlcov-tests --fail-under {env:TESTS_COV_MIN:96} \ + coverage html -d {envlogdir}/htmlcov-tests --fail-under {env:TESTS_COVERAGE_MIN:96} \ --include="tests/*" - diff-cover --compare-branch=origin/master \ - {env:PY3_DIFFCOVER_OPTIONS} --fail-under {env:DIFF_COV_MIN:92} \ + diff-cover --compare-branch=origin/master --exclude xcp/dmv.py \ + {env:PY3_DIFFCOVER_OPTIONS} --fail-under {env:DIFF_COVERAGE_MIN:92} \ --html-report {envlogdir}/coverage-diff.html \ {envlogdir}/coverage.xml