From 85129cb8d7d154f57eabb3bc060dab4baaf039fb Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Tue, 25 Jun 2019 15:27:18 +0200 Subject: [PATCH 1/4] Instruct travis to install coveralls Coveralls is used to publish coverage results online via coveralls.io. Travis is already configured to run it "after_success", but this has failed for a while, because it was not installed. Signed-off-by: Lukas Puehringer --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cc510b8208..a6b82b43d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ env: - TOXENV=py36 before_script: - - pip install pylint bandit tox + - pip install pylint bandit tox coveralls script: - pylint tuf From bd418968e5f2a616d1f86e33e1e8ec06f80f55ea Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Tue, 25 Jun 2019 15:30:51 +0200 Subject: [PATCH 2/4] Update travis to xenial and misc updates - Add a build matrix to run each tox env in a corresponding travis env as per travis/tox best practices. https://docs.travis-ci.com/user/languages/python/#using-tox-as-the-build-script - Add Python 3.5 tests - Remove only build on certain branch restrictions - Use "install" instead of "before_script" to install dependencies. Explicitly listing "install" prevents Travis from automatically running `pip install -r requirements.txt`, which is not necessary because most of those requirements are installed again in each tox environment. - Move pylint and bandit calls to tox (pylint requires dependencies) to be installed. Signed-off-by: Lukas Puehringer --- .travis.yml | 30 +++++++++++++----------------- ci-requirements.txt | 2 +- tox.ini | 2 ++ 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6b82b43d0..75f7ccf9bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,25 @@ +dist: xenial language: python -dist: trusty -sudo: false cache: pip -python: 3.6 -env: - - TOXENV=py27 - - TOXENV=py34 - - TOXENV=py36 +matrix: + include: + - python: "2.7" + env: TOXENV=py27 + - python: "3.4" + env: TOXENV=py34 + - python: "3.5" + env: TOXENV=py35 + - python: "3.6" + env: TOXENV=py36 -before_script: - - pip install pylint bandit tox coveralls +install: + - pip install tox coveralls script: - - pylint tuf - - bandit -r tuf - tox after_success: - cd tests - coveralls - cd - - -branches: - only: - - develop - - pylint - - bandit diff --git a/ci-requirements.txt b/ci-requirements.txt index f737ed8da2..ac26653c63 100644 --- a/ci-requirements.txt +++ b/ci-requirements.txt @@ -2,6 +2,6 @@ securesystemslib[crypto,pynacl] six iso8601 coverage -coveralls pylint +bandit requests diff --git a/tox.ini b/tox.ini index ddee62883c..f8609975d4 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ envlist = py27, py34, py35, py36 changedir = tests commands = + pylint {toxinidir}/tuf + bandit -r {toxinidir}/tuf coverage run --source tuf aggregate_tests.py coverage report -m --fail-under 97 From af18ead47379bfb2c4b9264272f9616d5b3b5a7b Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Tue, 25 Jun 2019 17:32:52 +0200 Subject: [PATCH 3/4] Pin coverage dependency for coveralls We install coverage inside tox builds to generate test coverage reports. These reports need to be created with a version supported by coveralls, which we use (outside of tox) to publish coverage reports to coveralls.io. Signed-off-by: Lukas Puehringer --- ci-requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-requirements.txt b/ci-requirements.txt index ac26653c63..e0ef0d81d6 100644 --- a/ci-requirements.txt +++ b/ci-requirements.txt @@ -1,7 +1,9 @@ securesystemslib[crypto,pynacl] six iso8601 -coverage +requests pylint bandit -requests +# Pin to versions supported by `coveralls` (see .travis.yml) +# https://github.com/coveralls-clients/coveralls-python/releases/tag/1.8.1 +coverage<5.0 From 157167e0cc8c3f9bd9ea8d96f1d796e08bfa932d Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 26 Jun 2019 16:54:19 +0200 Subject: [PATCH 4/4] Fix coverage file paths by patching sys.path This replicates behavior of unittest's `discover` method, and allows `coverage` and the tool that posts coverage reports to coveralls.io, i.e. `coveralls`, to record the correct paths and left-strip the parts leading to the project directory. Signed-off-by: Lukas Puehringer --- .travis.yml | 10 +++++++--- tests/.coveragerc | 2 ++ tests/aggregate_tests.py | 13 +++++++++++++ tox.ini | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75f7ccf9bb..f98055a040 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,10 @@ script: - tox after_success: - - cd tests - - coveralls - - cd - + # Workaround to get coverage reports with relative paths. + # FIXME: Consider refactoring the tests to not require the test aggregation + # script being invoked from the `tests` directory, so that `.coverage` is + # written to and .coveragrc can also reside in the project root directory, as + # as the convention. + - cp tests/.coverage . + - coveralls --rcfile=tests/.coveragerc diff --git a/tests/.coveragerc b/tests/.coveragerc index e45895f8cd..e9175732d6 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -11,3 +11,5 @@ omit = # Command-line scripts. */tuf/scripts/client.py */tuf/scripts/repo.py + */tests/* + */site-packages/* diff --git a/tests/aggregate_tests.py b/tests/aggregate_tests.py index d814737928..a0354f4600 100755 --- a/tests/aggregate_tests.py +++ b/tests/aggregate_tests.py @@ -34,6 +34,7 @@ from __future__ import division from __future__ import unicode_literals +import os import sys import unittest import glob @@ -90,9 +91,21 @@ random.shuffle(test_modules_to_run) if __name__ == '__main__': + # NOTE: Temporary workaround to ensure that `tuf` code from the sibling `tuf` + # directory is prioritized over `tuf` code installed to some system or + # virtualenv site-packages directory, which in turn is required for + # coverage/coveralls to function correctly. + # FIXME: Consider refactoring the tests to not require this the aggregation + # script being invoked from the `tests` directory. This seems to be the + # convention and would make use of other testing tools such as + # coverage/coveralls easier. + sys.path.insert(0, + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + suite = unittest.TestLoader().loadTestsFromNames(test_modules_to_run) all_tests_passed = unittest.TextTestRunner( verbosity=1, buffer=True).run(suite).wasSuccessful() + if not all_tests_passed: sys.exit(1) diff --git a/tox.ini b/tox.ini index f8609975d4..79bc9a6abd 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ changedir = tests commands = pylint {toxinidir}/tuf bandit -r {toxinidir}/tuf - coverage run --source tuf aggregate_tests.py + coverage run aggregate_tests.py coverage report -m --fail-under 97 deps =