Refactor --coverage in the test runner

.travis.yml

@@ -30,7 +30,7 @@ env:
         - MAIN_CMD='python setup.py'
         - SETUP_CMD='test --coverage'
         - CONDA_CHANNELS='sunpy'
-        - CONDA_DEPENDENCIES='openjpeg Cython jinja2 scipy matplotlib mock requests beautifulsoup4 sqlalchemy scikit-image pytest-mock lxml pyyaml pandas nomkl pytest-astropy suds-jurko glymur'
+        - CONDA_DEPENDENCIES='openjpeg Cython jinja2 scipy matplotlib mock requests beautifulsoup4 sqlalchemy scikit-image pytest-mock lxml pyyaml pandas nomkl pytest-astropy suds-jurko glymur pytest-xdist'
         - PIP_DEPENDENCIES='git+https://github.com/sphinx-gallery/sphinx-gallery sphinx-astropy pytest-sugar pytest-rerunfailures pytest-cov sunpy-sphinx-theme hypothesis drms'
         - EVENT_TYPE='push pull_request cron'
         - MPLBACKEND='agg'
@@ -59,7 +59,7 @@ matrix:
         # We run --online here because --online-only skips the doctests
          - python: 3.6
            stage: Comprehensive tests
-           env: SETUP_CMD="test --online --coverage"
+           env: SETUP_CMD="test --online --coverage --parallel 8"
 
         # Cron tests below this line
          - os: osx

changelog/2667.trivial.rst

@@ -0,0 +1 @@
+Change to using pytest-cov for coverage report generation to enable support for parallel builds

sunpy/tests/coveragerc

@@ -1,15 +1,16 @@
 [run]
 source = sunpy
 omit =
-   sunpy/_astropy_init*
    sunpy/conftest*
    sunpy/cython_version*
    sunpy/*setup*
-   sunpy/*tests/*
+   sunpy/extern/*
+   sunpy/*/tests/*
    sunpy/version*
    sunpy/__init__*
 
 [report]
+
 exclude_lines =
    # Have to re-enable the standard pragma
    pragma: no cover

sunpy/tests/helpers.py

@@ -104,3 +104,42 @@ def wrapper(*args, **kwargs):
                                'New image generated and placed at {}'.format(result_image_loc))
 
     return wrapper
+
+
+# Skip coverage on this because we test it every time the CI runs --coverage!
+def _patch_coverage(testdir, sourcedir):  # pragma: no cover
+    """
+    This function is used by the ``setup.py test`` command to change the
+    filepath of the source code from the temporary directory setup.py installs
+    the code into to the actual directory setup.py was executed in.
+    """
+    import coverage
+
+    coveragerc = os.path.join(os.path.dirname(__file__), "coveragerc")
+
+    # Load the .coverage file output by pytest-cov
+    covfile = os.path.join(testdir, ".coverage")
+    cov = coverage.Coverage(covfile, config_file=coveragerc)
+    cov.load()
+    cov.get_data()
+
+    # Change the filename for the datafile to the new directory
+    if hasattr(cov, "_data_files"):
+        dfs = cov._data_files
+    else:
+        dfs = cov.data_files
+
+    dfs.filename = os.path.join(sourcedir, ".coverage")
+
+    # Replace the testdir with source dir
+    # Lovingly borrowed from astropy (see licences directory)
+    lines = cov.data._lines
+    for key in list(lines.keys()):
+        new_path = os.path.relpath(
+            os.path.realpath(key),
+            os.path.realpath(testdir))
+        new_path = os.path.abspath(
+            os.path.join(sourcedir, new_path))
+        lines[new_path] = lines.pop(key)
+
+    cov.save()

sunpy/tests/runner.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
+import os
+
 from astropy.tests.runner import TestRunner, keyword
 
 
@@ -12,7 +14,7 @@ class SunPyTestRunner(TestRunner):
     # Disable certain astropy flags
     @keyword()
     def remote_data(self, remote_data, kwargs):
-        return NotImplemented
+        return NotImplemented  # pragma: no cover
 
     # Change the docsting on package
     @keyword(priority=10)
@@ -94,3 +96,22 @@ def plugins(self, plugins, kwargs):
         # Plugins are handled independently by `run_tests` so we define this
         # keyword just for the docstring
         return []
+
+    @keyword()
+    def coverage(self, coverage, kwargs):
+        if coverage:
+            coveragerc = os.path.join(self.base_path, "tests", "coveragerc")
+            ret = []
+            for path in self.package_path:
+                ret += ["--cov", path, "--cov-config", coveragerc]
+            return ret
+
+        return []
+
+    @keyword()
+    def cov_report(self, cov_report, kwargs):
+        if kwargs['coverage'] and cov_report:
+            a = [cov_report] if isinstance(cov_report, str) else []
+            return ['--cov-report'] + a
+
+        return []

sunpy/tests/setup_command.py

@@ -22,6 +22,8 @@ class SunPyTest(AstropyTest):
                       'Also run tests that do require a internet connection.'),
                      ('online-only', None,
                       'Only run test that do require a internet connection.'),
+                     ('cov-report=', None,
+                      'How to display the coverage report, should be either "html" or "term"'),
                      ('figure', None,
                       'Run the figure tests.'),
                      # Run only tests that check figure generation
@@ -36,6 +38,28 @@ def initialize_options(self):
         self.online_only = False
         self.figure = False
         self.figure_only = False
+        self.cov_report = True
+
+    def _generate_coverage_commands(self):
+        cmd_pre = ''  # Commands to run before the test function
+
+        # patch the .coverage file so the paths are correct to the directory
+        # setup.py was run in rather than the temporary directory.
+        cwd = os.path.abspath(".")
+        cmd_post = ('from sunpy.tests.helpers import _patch_coverage; '
+                    'import os; '
+                    'test_dir = os.path.abspath("."); '
+                    f'_patch_coverage(test_dir, "{cwd}"); ')
+
+        # Make html report the default and make pytest-cov save it to the
+        # source directory not the temporary directory.
+        if self.cov_report and (isinstance(self.cov_report, bool) or "html" in self.cov_report):
+            html_cov = os.path.join(os.path.abspath("."), "htmlcov")
+            self.cov_report = f'html:{html_cov}'
+        else:
+            self.cov_report = self.cov_report
+
+        return cmd_pre, cmd_post
 
     def generate_testing_command(self):
         """
@@ -55,6 +79,8 @@ def generate_testing_command(self):
                'package={1.package!r}, '
                'test_path={1.test_path!r}, '
                'args={1.args!r}, '
+               'coverage={1.coverage!r}, '
+               'cov_report={1.cov_report!r}, '
                'plugins={1.plugins!r}, '
                'verbose={1.verbose_results!r}, '
                'pastebin={1.pastebin!r}, '

sunpy/tests/tests/test_self_test.py

@@ -1,12 +1,25 @@
+import imp
 import os.path
 
 import pytest
 
 import sunpy
+import sunpy.tests.runner
+
 
 root_dir = os.path.dirname(os.path.abspath(sunpy.__file__))
 
 
+def test_import_runner():
+    """
+    When running the tests with setup.py, the test runner class is imported by
+    setup.py before coverage is watching. To ensure that the coverage for
+    sunpy/tests/runner.py is correctly measured we force the interpreter to
+    reload it here while coverage is watching.
+    """
+    imp.reload(sunpy.tests.runner)
+
+
 def test_main_nonexisting_module():
     with pytest.raises(ValueError):
         sunpy.self_test(package='doesnotexist')
@@ -49,6 +62,13 @@ def test_main_exclude_remote_data(monkeypatch):
                     [os.path.join(root_dir, 'map'), '-m', 'not figure'])
 
 
+def test_main_include_remote_data(monkeypatch):
+    monkeypatch.setattr(pytest, 'main', lambda args, **kwargs: args)
+    args = sunpy.self_test(package='map', online=True)
+    assert args in ([os.path.join('sunpy', 'map'), '--remote-data=any', '-m', 'not figure'],
+                    [os.path.join(root_dir, 'map'),'--remote-data=any',  '-m', 'not figure'])
+
+
 def test_main_only_remote_data(monkeypatch):
     monkeypatch.setattr(pytest, 'main', lambda args, **kwargs: args)
     args = sunpy.self_test(package='map', online_only=True)
@@ -68,3 +88,42 @@ def test_main_figure_only(monkeypatch):
     args = sunpy.self_test(figure_only=True)
     assert args in (['sunpy', '-m', 'figure'],
                     [root_dir, '-m', 'figure'])
+
+
+def test_main_figure_dir(monkeypatch):
+    monkeypatch.setattr(pytest, 'main', lambda args, **kwargs: args)
+    args = sunpy.self_test(figure_only=True, figure_dir=".")
+    assert args in (['sunpy', '--figure_dir', '.', '-m', 'figure'],
+                    [root_dir, '--figure_dir', '.', '-m', 'figure'])
+
+
+def test_main_coverage(monkeypatch):
+    monkeypatch.setattr(pytest, 'main', lambda args, **kwargs: args)
+    args = sunpy.self_test(coverage=True)
+    for a in args:
+        assert a in [root_dir, 'sunpy', '--cov', '--cov-config', '-m',
+                     'not figure',
+                     os.path.join(root_dir, 'tests', 'coveragerc'),
+                     os.path.join('sunpy', 'tests', 'coveragerc')]
+
+
+def test_main_coverage_report(monkeypatch):
+    monkeypatch.setattr(pytest, 'main', lambda args, **kwargs: args)
+    args = sunpy.self_test(coverage=True, cov_report=True)
+    for a in args:
+        assert a in [root_dir, 'sunpy', '--cov', '--cov-config', '-m',
+                     'not figure',
+                     os.path.join(root_dir, 'tests', 'coveragerc'),
+                     os.path.join('sunpy', 'tests', 'coveragerc'),
+                     '--cov-report']
+
+
+def test_main_coverage_report_html(monkeypatch):
+    monkeypatch.setattr(pytest, 'main', lambda args, **kwargs: args)
+    args = sunpy.self_test(coverage=True, cov_report=True)
+    for a in args:
+        assert a in [root_dir, 'sunpy', '--cov', '--cov-config', '-m',
+                     'not figure',
+                     os.path.join(root_dir, 'tests', 'coveragerc'),
+                     os.path.join('sunpy', 'tests', 'coveragerc'),
+                     '--cov-report']