Merge pull request #1129 from glyph/enhanced-coverage-configuration

Author: glyph

Reviewer: glyph

Fixes: ticket:9625

This uploads coverage data to both codecov and coveralls in order for us to compare the present state of the art on these two services, in a way which is (hopefully) backwards compatible to buildbot environments that are a little trickier to update.

This leaves codecov receiving coverage from all buildbots, but uploads coveralls only from our cloud CI environments, hopefully providing us with a more stable coverage metric that contributors can score themselves against.

.circleci/config.yml

@@ -148,6 +148,11 @@ jobs:
           command: |
             tox -r -e py27-alldeps-withcov-posix twisted
 
+      - run:
+          name: Upload coverage.
+          command: |
+            tox -r -e coverage-prepare,codecov-push,coveralls-push
+
   #
   # MacOS with Python3.7 and default reactor.
   #
@@ -175,6 +180,11 @@ jobs:
           command: |
             tox -r -e py37-alldeps-withcov-posix twisted
 
+      - run:
+          name: Upload coverage.
+          command: |
+            tox -r -e coverage-prepare,codecov-push,coveralls-push
+
 
 # First we run the static checkers, and only if they pass we spin the macOS.
 # in this way we should save some macOS minutes as we only have 1000h per month.

.coveralls.yml

@@ -0,0 +1 @@
+repo_token: "JFDTIRUVOQ8jCM3zcajrZALlpKXyiXGAX"

.travis.yml

@@ -27,21 +27,21 @@ matrix:
         - docker
       env: PIP=pip TOXENV=wheels
     - python: 2.7
-      env: TOXENV=py27-alldeps-withcov-posix,codecov-publish
+      env: TOXENV=py27-alldeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push
     - python: 2.7
-      env: TOXENV=py27-nodeps-withcov-posix,codecov-publish
+      env: TOXENV=py27-nodeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push
     - python: 3.5
-      env: TOXENV=py35-alldeps-withcov-posix,codecov-publish
+      env: TOXENV=py35-alldeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push
     - python: 3.6
-      env: TOXENV=py36-alldeps-withcov-posix,codecov-publish
+      env: TOXENV=py36-alldeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push
     - python: 3.7
-      env: TOXENV=py37-alldeps-withcov-posix,codecov-publish
+      env: TOXENV=py37-alldeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push
     - python: 3.5
       env: TOXENV=lint
     # We need a builder without IPv6. This is going to be slower than all the
     # others, but that's ok.
     - python: 3.6
-      env: TOXENV=py36-alldeps-withcov-posix,codecov-publish DISABLE_IPV6=yes
+      env: TOXENV=py36-alldeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push DISABLE_IPV6=yes
 
 addons:
   apt:

.travis/run.sh

@@ -6,7 +6,7 @@ if [[ "$(uname -s)" == "Darwin" ]]; then
     # Initialize the virtualenv created at install time.
     source ~/.venv/bin/activate
 
-    if [[ "${TOXENV}" == "py35-alldeps-withcov-macos,codecov-publish" ]]; then
+    if [[ "${TOXENV}" =~ py35-.* ]]; then
         # Add pyenv path
         PYENV_ROOT="$HOME/.pyenv";
         PATH="$PYENV_ROOT/bin:$PATH";

MANIFEST.in

@@ -20,6 +20,7 @@ include NEWS.rst README.rst INSTALL.rst CONTRIBUTING LICENSE code_of_conduct.md
 exclude pyproject.toml
 exclude codecov.yml
 exclude appveyor.yml
+exclude .coveralls.yml
 exclude .circleci
 recursive-exclude .circleci *
 prune bin

appveyor.yml

@@ -34,49 +34,49 @@ environment:
   # a later point release.
   # See: http://www.appveyor.com/docs/installed-software#python
   matrix:
-    - PYTHON_HOME: C:\\PYTHON36-x64
-      PYTHON_VERSION: "3.6"
-      PYTHON_ARCH: "64"
-      TOXENV: wheels
-
     - PYTHON_HOME: C:\\Python27-x64
       PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
-      TOXENV: py27-alldeps-withcov-windows,codecov-publish
+      TOXENV: py27-alldeps-withcov-windows,coverage-prepare,codecov-push,coveralls-push
       TWISTED_REACTOR: "select"
     - PYTHON_HOME: C:\\PYTHON36-x64
       PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
-      TOXENV: py36-alldeps-withcov-windows,codecov-publish
+      TOXENV: py36-alldeps-withcov-windows,coverage-prepare,codecov-push,coveralls-push
       TWISTED_REACTOR: "select"
     - PYTHON_HOME: C:\\PYTHON37-x64
       PYTHON_VERSION: "3.7"
       PYTHON_ARCH: "64"
-      TOXENV: py37-alldeps-withcov-windows,codecov-publish
+      TOXENV: py37-alldeps-withcov-windows,coverage-prepare,codecov-push,coveralls-push
       TWISTED_REACTOR: "select"
     - PYTHON_HOME: C:\\PYTHON37-x64
       PYTHON_VERSION: "3.7"
       PYTHON_ARCH: "64"
-      TOXENV: py37-nodeps-withcov-windows,codecov-publish
+      TOXENV: py37-nodeps-withcov-windows,coverage-prepare,codecov-push,coveralls-push
       TWISTED_REACTOR: "select"
 
     # IOCPReactor tests
     - PYTHON_HOME: C:\\Python27-x64
       PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
-      TOXENV: py27-alldeps-withcov-windows,codecov-publish
+      TOXENV: py27-alldeps-withcov-windows,coverage-prepare,codecov-push,coveralls-push
       TWISTED_REACTOR: "iocp"
     - PYTHON_HOME: C:\\PYTHON36-x64
       PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
-      TOXENV: py36-alldeps-withcov-windows,codecov-publish
+      TOXENV: py36-alldeps-withcov-windows,coverage-prepare,codecov-push,coveralls-push
       TWISTED_REACTOR: "iocp"
     - PYTHON_HOME: C:\\PYTHON37-x64
       PYTHON_VERSION: "3.7"
       PYTHON_ARCH: "64"
-      TOXENV: py37-alldeps-withcov-windows,codecov-publish
+      TOXENV: py37-alldeps-withcov-windows,coverage-prepare,codecov-push,coveralls-push
       TWISTED_REACTOR: "iocp"
 
+    - PYTHON_HOME: C:\\PYTHON36-x64
+      PYTHON_VERSION: "3.6"
+      PYTHON_ARCH: "64"
+      TOXENV: wheels
+
 
 install:
   # Install OpenSSH through chocolatey

codecov.yml

@@ -12,6 +12,4 @@ coverage:
       default:
         target: 100%
 
-comment:
-  layout: "header, diff, uncovered"
-  behavior: default
+comment: off

tox.ini

@@ -13,17 +13,13 @@
 ; * macos1010 - prepare the tests to be run in a macOS 10.10 environment (superset of posix)
 ; * windows - prepare the tests to be executed under Windows.
 ;
-; Some tags are defined only while transitioning to an updated Buildbot and
-; should be removed after the transition.
-; See: https://github.com/twisted-infra/braid/issues/222
-;
-; * tests - similar to alldeps
-; * coverage - similar to withcov and alldeps
-; * nomodules - similar to nodeps
-;
-; There is a non-default `codecov-publish` environment used to publish the
-; coverage results to codecov.io. It should be called after running the
-; standard coverage environment.
+; There is are non-default `codecov-push` and `coveralls-push` environments
+; used to push the coverage results to codecov.io and coveralls.io,
+; respectively. They should be called after running both some number of
+; `-withcov` environments and also `coverage-prepare`.
+
+; For compatibility with the current buildbot infrastructure, `codecov-publish`
+; combines `coverage-prepare` and `codecov-push` into a single step.
 ;
 ; A non-default `txchecker-travis` environment is used to run twistedchecker
 ; on travis in --diff mode
@@ -37,22 +33,27 @@ envlist=lint,pyflakes,apidocs,narrativedocs,newsfragment,manifest-checker,py27-a
 [testenv]
 ;; dependencies managed by extras in t.p._setup.py._EXTRAS_REQUIRE
 extras =
-    {alldeps,tests,coverage}: all_non_platform
+    alldeps: all_non_platform
 
     windows: windows_platform
 
-    {alldeps,tests}-macos: osx_platform
+    alldeps-macos: osx_platform
 
     ; Documentation needs Twisted install to get the version.
     narrativedocs: dev
 
 ;; dependencies that are not specified as extras
 deps =
-    py27-{alldeps,tests,coverage}-{posix,macos}: pysqlite
+    py27-alldeps-{posix,macos}: pysqlite
 
-    {withcov,coverage}: coverage
+    {withcov}: coverage
 
-    codecov-publish: codecov==2.0.5
+    {coverage-prepare,codecov-publish}: coverage
+
+    {codecov-push,codecov-publish}: codecov
+
+    coveralls-push: coveralls
+    coveralls-push: PyYAML
 
     wheel: wheel
 
@@ -99,17 +100,22 @@ commands =
     apidocs: pip install --no-deps epydoc pydoctor
 
     ; Run tests without wrapping them using coverage.
-    {nocov,nomodules,tests}: python -m twisted.trial --reactor={env:TWISTED_REACTOR:default} --reporter={env:TRIAL_REPORTER:verbose}  {posargs:twisted}
+    nocov: python -m twisted.trial --reactor={env:TWISTED_REACTOR:default} --reporter={env:TRIAL_REPORTER:verbose}  {posargs:twisted}
 
     ; Run the tests wrapped using coverage.
-    {withcov,coverage}: python {toxinidir}/admin/_copy.py {toxinidir}/admin/zz_coverage.pth {envsitepackagesdir}/zz_coverage.pth
-    {withcov,coverage}: coverage erase
-    {withcov,coverage}: coverage run -p --rcfile={toxinidir}/.coveragerc -m twisted.trial --reactor={env:TWISTED_REACTOR:default} --reporter={env:TRIAL_REPORTER:verbose} {posargs:twisted}
+    withcov: python {toxinidir}/admin/_copy.py {toxinidir}/admin/zz_coverage.pth {envsitepackagesdir}/zz_coverage.pth
+    withcov: coverage erase
+    withcov: coverage run -p --rcfile={toxinidir}/.coveragerc -m twisted.trial --reactor={env:TWISTED_REACTOR:default} --reporter={env:TRIAL_REPORTER:verbose} {posargs:twisted}
+
+    ; Prepare coverage reports for publication.
+    {coverage-prepare,codecov-publish}: coverage combine
+    {coverage-prepare,codecov-publish}: coverage xml -o coverage.xml -i
 
-    ; Publish the coverage reports to codecov.io
-    codecov-publish: coverage combine
-    codecov-publish: coverage xml -o coverage.xml -i
-    codecov-publish: codecov {env:CODECOV_OPTIONS:} -X search -X gcov -f coverage.xml
+    ; Publish coverage reports to codecov.
+    {codecov-push,codecov-publish}: codecov {env:CODECOV_OPTIONS:} -X search -X gcov -f coverage.xml
+
+    ; Publish coverage reports to coveralls.
+    coveralls-push: coveralls
 
     ; Build in a root folder so that buildbot can pick the result
     wheel: python setup.py bdist_wheel --dist-dir {toxinidir}/dist
@@ -140,3 +146,15 @@ skip_install = true
 [testenv:lint]
 basepython=python3.5
 usedevelop=True
+
+[testenv:coverage-prepare]
+skip_install=True
+
+[testenv:codecov-publish]
+skip_install=True
+
+[testenv:codecov-push]
+skip_install=True
+
+[testenv:coveralls-push]
+skip_install=True