Merge pull request #1574 from twisted/10159-codecov-pr-fix

Author: adiroiban
Reviewer: altendky 
Fixes: ticket:10159

Fix codecov reporting for a PR.
Move coverage CI reporting outside of tox.
Normalize code coverage path reporting inside tox environments.

.coveragerc

@@ -2,13 +2,14 @@
 branch = True
 parallel = True
 source = twisted
+omit =
+    .tox/*/tmp/_trial_temp/*
 
 [paths]
 source=
    src/twisted
-   build/*/lib/python*/site-packages/twisted
-   build/*/Lib/site-packages/twisted
-   build/pypy*/site-packages/twisted
+   */site-packages/twisted
+   *\site-packages\twisted
 
 [report]
 precision = 2

.github/workflows/test.yaml

@@ -21,7 +21,6 @@ jobs:
     runs-on: ubuntu-20.04
     env:
       TOXENV: "${{ matrix.tox-env }}"
-      CODECOV_OPTIONS: "-n 'lnx-${{ matrix.python-version }}-${{ matrix.tox-env }}${{ matrix.noipv6 }}'"
       TRIAL_ARGS: "${{ matrix.trial-args }}"
     name: ${{ matrix.python-version }}${{ matrix.noipv6 }}-${{ matrix.tox-env }}
     strategy:
@@ -48,13 +47,14 @@ jobs:
           # end to end functional test usage for non-distributed trial runs.
           - python-version: 3.6.7
             tox-env: nodeps-withcov-posix
-            trial-args: ''
+            trial-args: '-j 4'
           # `noipv6` is created to make sure all is OK on an OS which doesn't
           # have IPv6 available.
           # Any supported Python version is OK for this job.
           - python-version: 3.6
             tox-env: alldeps-withcov-posix
             noipv6: -noipv6
+            trial-args: '-j 4'
           # On PYPY concurrent test jobs result in random failures so for now
           # run non-distributed tests.
           - python-version: pypy-3.6
@@ -99,18 +99,41 @@ jobs:
     - uses: twisted/python-info-action@v1
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip tox
+        python -m pip install --upgrade pip tox coveralls
         tox --notest
 
     - name: Test
       run: |
         python --version
         tox -q
 
-    - name: Publish coverage
-      # We want to publish coverage even on failure.
-      if: contains(matrix['tox-env'], 'withcov') || failure()
-      run: tox -e coverage-prepare,codecov-push,coveralls-push
+    - name: Prepare coverage
+      if: always()
+      continue-on-error: true
+      run: |
+        # sub-process coverage are generated in separate files so we combine them
+        # to get an unified coverage for the local run.
+        # The XML is generate to be used with 3rd party tools like diff-cover.
+        python -m coverage combine
+        python -m coverage xml -o coverage.xml -i
+        python -m coverage report
+
+    - uses: codecov/codecov-action@v1
+      if: always()
+      continue-on-error: true
+      with:
+        files: coverage.xml
+        name: lnx-${{ matrix.python-version }}-${{ matrix.tox-env }}${{ matrix.noipv6 }}
+        fail_ci_if_error: true
+        functionalities: gcov,search
+
+    - name: Publish to Coveralls
+      if: always()
+      continue-on-error: true
+      run: |
+        python -m coveralls -v
+      env:
+        COVERALLS_REPO_TOKEN: 'JFDTIRUVOQ8jCM3zcajrZALlpKXyiXGAX'
 
 
   pypi-publish:

azure-pipelines/run_test_steps.yml

@@ -28,7 +28,9 @@ parameters:
 steps:
 - checkout: self
   clean: true
-  fetchDepth: 1
+  # A minimum depth of 2 is required, as the HEAD is the auto-merge commit
+  # and we need the last commit from the PR to report the coverage.
+  fetchDepth: 2
 
 - task: UsePythonVersion@0
   displayName: "Use Python ${{ parameters.pythonVersion }}"
@@ -45,7 +47,7 @@ steps:
    python -c "import os; [ print(e,v) for (e,v) in os.environ.items() ]"
   displayName: 'Get Python Information'
 
-- script: 'python -m pip install -U pip setuptools tox'
+- script: 'python -m pip install -U pip setuptools tox coverage coveralls'
   displayName: 'Update pip & install tox'
 
 - ${{ if eq(parameters.platform, 'macos') }}:
@@ -62,7 +64,39 @@ steps:
     env:
       TWISTED_REACTOR: ${{ parameters.windowsReactor }}
 
-- bash:  |
-    python -m tox -e coverage-prepare,coveralls-push
-    bash <(curl -s https://codecov.io/bash) -n "${{ parameters.platform }}-${{ parameters.pythonVersion }}-alldeps-withcov"
-  displayName: 'Report coverage'
+- bash: |
+    # sub-process coverage are generated in separate files so we combine them
+    # to get an unified coverage for the local run.
+    # The XML is generate to be used with 3rd party tools like diff-cover.
+    python -m coverage combine
+    python -m coverage xml -o coverage.xml -i
+    python -m coverage report
+
+  displayName: 'Prepare coverage'
+  condition: always()
+  continueOnError: true
+
+- bash: |
+    bash <(curl -s https://codecov.io/bash) -X search -X gcov -f coverage.xml -n "${{ parameters.platform }}-${{ parameters.pythonVersion }}"
+  displayName: 'Report to Codecov'
+  condition: always()
+  continueOnError: true
+
+# We are using a 3rd part tools to upload to Coveralls
+# See https://github.com/TheKevJames/coveralls-python
+# It has no support for Azure so we pretend to be CircleCI so that
+# we can differentiate from GitHub Actions.
+# https://github.com/TheKevJames/coveralls-python/blob/04b6a2876e4e7ab2e8cf0778f88ce23f94679931/coveralls/api.py#L147
+# https://github.com/TheKevJames/coveralls-python/blob/04b6a2876e4e7ab2e8cf0778f88ce23f94679931/coveralls/git.py#L31
+- bash: |
+    export CI_BRANCH=$SYSTEM_PULLREQUEST_SOURCEBRANCH
+    export CI_PULL_REQUEST=$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER
+    export CI_BUILD_URL=$SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI/pull/$CI_PULL_REQUEST
+    python -m coveralls -v
+  displayName: 'Report to Coveralls'
+  condition: always()
+  continueOnError: true
+  env:
+    CIRCLE_WORKFLOW_ID: $(Build.BuildNumber)
+    CIRCLECI: 1
+    COVERALLS_REPO_TOKEN: 'JFDTIRUVOQ8jCM3zcajrZALlpKXyiXGAX'

tox.ini

@@ -15,15 +15,6 @@
 ;
 ; See README.rst for example tox commands.
 ;
-; There are also various non-default environments used by the continuous
-; integration system: the `codecov-push` and `coveralls-push` push the coverage
-; results to codecov.io and coveralls.io, respectively. They should be called
-; after running both some number of `-withcov` environments and also
-; `coverage-prepare`.
-;
-; For compatibility with the current infrastructure, `codecov-publish`
-; combines `coverage-prepare` and `codecov-push` into a single step.
-;
 [tox]
 minversion=3.21.4
 requires=
@@ -60,20 +51,10 @@ extras =
 
     serial: serial
 
-    {withcov,coverage-prepare,codecov-publish}: dev
+    {withcov}: dev
 
 ;; dependencies that are not specified as extras
 deps =
-    ; We end up with a bit of deps duplication as we can't install
-    ; coverage deps via `extras` as we run them with `skip_install`.
-    {coverage-prepare,codecov-publish}: coverage ~= 5.5
-
-    {codecov-push,codecov-publish}: codecov ~= 2.1
-
-    coveralls-push: coveralls
-    coveralls-push: PyYAML
-
-
     lint: pre-commit
 
 ; All environment variables are passed.
@@ -90,9 +71,6 @@ setenv =
     {windows,serial}: TWISTED_FORCE_SERIAL_TESTS = 1
 
 skip_install =
-    coverage-prepare: True
-    codecov: True
-    coveralls: True
     lint: True
 
 commands =
@@ -112,16 +90,6 @@ commands =
     withcov: coverage erase
     withcov: coverage run -p --rcfile={toxinidir}/.coveragerc -m twisted.trial --temp-directory={envtmpdir}/_trial_temp --reactor={env:TWISTED_REACTOR:default} --reporter={env:TRIAL_REPORTER:verbose} {env:TRIAL_ARGS:} {posargs:twisted}
 
-    ; Prepare coverage reports for publication.
-    {coverage-prepare,codecov-publish}: coverage combine
-    {coverage-prepare,codecov-publish}: coverage xml -o coverage.xml -i
-
-    ; Publish coverage reports to codecov.
-    {codecov-push,codecov-publish}: codecov {env:CODECOV_OPTIONS:} -X search -X gcov -f coverage.xml
-
-    ; Publish coverage reports to coveralls.
-    coveralls-push: coveralls
-
     lint: pre-commit {posargs:run --all-files --show-diff-on-failure}
 
     newsfragment: python {toxinidir}/bin/admin/check-newsfragment "{toxinidir}"