Coverage reporting (#691)

* feature: add code coverage reporting

uses codecov

* feature: add coveralls support

* feature: generate coverage reports with make

Also makes setuptools install cov-pytest,
right now coverage is only generated by unit tests

* maintenance: remove coveralls support

To avoid failure states

* maintenance: add back rules for coverage reports

* maintenance: ignore make-generated files

* fix: appveyor broken pip install

Follow recommendation on https://pythonhosted.org/CodeChat/appveyor.yml.html

.coveragerc

@@ -1,3 +1,13 @@
 [run]
 branch = True
 source = cwltool
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+ignore_errors = True
+omit =
+    tests/*

.gitignore

@@ -30,3 +30,16 @@ typeshed/2and3/ruamel/yaml
 
 #mypy
 .mypy_cache/
+
+# Files generated by Makefile
+.cache/
+cache/
+.coverage
+coverage.xml
+htmlcov
+output
+output.txt
+pydocstyle_report.txt
+response.txt
+test.txt
+value

Makefile

@@ -113,23 +113,18 @@ pylint_report.txt: ${PYSOURCES}
 diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
 
-.coverage: $(PYSOURCES) all
-	export COVERAGE_PROCESS_START=${mkfile_dir}.coveragerc; \
-	       cd ${CWL}; ./run_test.sh RUNNER=cwltool
-	coverage run setup.py test
-	coverage combine ${CWL} ${CWL}/draft-3/ ./
+.coverage: tests
+
+coverage: .coverage
+	coverage report
 
 coverage.xml: .coverage
-	python-coverage xml
+	coverage xml
 
 coverage.html: htmlcov/index.html
 
 htmlcov/index.html: .coverage
-	python-coverage html
-	@echo Test coverage of the Python code is now in htmlcov/index.html
-
-coverage-report: .coverage
-	python-coverage report
+	coverage html
 
 diff-cover: coverage-gcovr.xml coverage.xml
 	diff-cover coverage-gcovr.xml coverage.xml
@@ -139,8 +134,8 @@ diff-cover.html: coverage-gcovr.xml coverage.xml
 		--html-report diff-cover.html
 
 ## test        : run the ${MODULE} test suite
-test: FORCE
-	./setup.py test
+test: $(PYSOURCES)
+	python setup.py test --addopts "--cov cwltool"
 
 sloccount.sc: ${PYSOURCES} Makefile
 	sloccount --duplicates --wide --details $^ > sloccount.sc

appveyor.yml

@@ -27,7 +27,7 @@ install:
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 
 build_script:
-  - "%CMD_IN_ENV% pip install -U setuptools"
+  - "%CMD_IN_ENV% python -m pip install -U setuptools"
   - "%CMD_IN_ENV% pip install ."
 
 

setup.py

@@ -16,7 +16,7 @@
     tagger = egg_info_cmd.egg_info
 
 needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv)
-pytest_runner = ['pytest-runner'] if needs_pytest else []
+pytest_runner = ['pytest-runner', 'pytest-cov'] if needs_pytest else []
 
 setup(name='cwltool',
       version='1.0',

tox.ini

@@ -17,12 +17,15 @@ skip_missing_interpreters = True
 3.6 = py36
 
 [testenv]
+passenv = CI TRAVIS TRAVIS_*
 deps =
   -rrequirements.txt
+  py{27,34,35,36}-unit: codecov
   py{27,34,35,36}-lint: flake8
 
 commands =
-  py{27,34,35,36}-unit: python setup.py test
+  py{27,34,35,36}-unit: python setup.py test --addopts "--cov-report xml --cov cwltool"
+  py{27,34,35,36}-unit: codecov
   py{27,34,35,36}-lint: flake8 schema_salad setup.py
 
 whitelist_externals =