Add coverage reporting (#2293)

Adds coverage reporting to CI, uploaded to codecov.io.  To get coverage locally, use `./runtests.py --coverage`.

.gitignore

@@ -23,3 +23,7 @@ docs/build/
 
 # Operating Systems
 .DS_store
+
+# Coverage Files
+htmlcov
+.coverage*

.travis.yml

@@ -13,4 +13,9 @@ install:
   - python setup.py install
 
 script:
-  - python runtests.py
+  - coverage erase
+  - python runtests.py --coverage
+  - coverage combine
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

conftest.py

@@ -1,3 +1,4 @@
 pytest_plugins = [
     'mypy.test.data',
+    'pytest_cov',
 ]

pytest.ini

@@ -11,4 +11,4 @@ python_classes =
 python_functions =
 
 # always run in parallel (requires pytest-xdist, see test-requirements.txt)
-addopts = -nauto
+addopts = -nauto --cov-append --cov-report=

runtests.py

@@ -40,7 +40,7 @@ class Driver:
 
     def __init__(self, whitelist: List[str], blacklist: List[str],
             arglist: List[str], verbosity: int, parallel_limit: int,
-            xfail: List[str]) -> None:
+            xfail: List[str], coverage: bool) -> None:
         self.whitelist = whitelist
         self.blacklist = blacklist
         self.arglist = arglist
@@ -50,6 +50,7 @@ def __init__(self, whitelist: List[str], blacklist: List[str],
         self.cwd = os.getcwd()
         self.mypy = os.path.join(self.cwd, 'scripts', 'mypy')
         self.env = dict(os.environ)
+        self.coverage = coverage
 
     def prepend_path(self, name: str, paths: List[str]) -> None:
         old_val = self.env.get(name)
@@ -94,11 +95,15 @@ def add_mypy_package(self, name: str, packagename: str, *flags: str) -> None:
     def add_mypy_string(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
         self.add_mypy_cmd(name, ['-c'] + list(args), cwd=cwd)
 
-    def add_pytest(self, name: str, pytest_args: List[str]) -> None:
+    def add_pytest(self, name: str, pytest_args: List[str], coverage: bool = False) -> None:
         full_name = 'pytest %s' % name
         if not self.allow(full_name):
             return
-        args = [sys.executable, '-m', 'pytest'] + pytest_args
+        if coverage and self.coverage:
+            args = [sys.executable, '-m', 'pytest', '--cov=mypy'] + pytest_args
+        else:
+            args = [sys.executable, '-m', 'pytest'] + pytest_args
+
         self.waiter.add(LazySubprocess(full_name, args, env=self.env))
 
     def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
@@ -110,12 +115,16 @@ def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
         env = self.env
         self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env))
 
-    def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
+    def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None,
+                       coverage: bool = False) -> None:
         name = 'run %s' % name
         if not self.allow(name):
             return
         largs = list(args)
-        largs[0:0] = [sys.executable, '-m']
+        if coverage and self.coverage:
+            largs[0:0] = ['coverage', 'run', '-m']
+        else:
+            largs[0:0] = [sys.executable, '-m']
         env = self.env
         self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env))
 
@@ -203,7 +212,7 @@ def add_imports(driver: Driver) -> None:
 
 def add_pytest(driver: Driver) -> None:
     for f in PYTEST_FILES:
-        driver.add_pytest(f, [f] + driver.arglist)
+        driver.add_pytest(f, [f] + driver.arglist, True)
 
 
 def add_myunit(driver: Driver) -> None:
@@ -218,17 +227,20 @@ def add_myunit(driver: Driver) -> None:
             # This module has been converted to pytest; don't try to use myunit.
             pass
         else:
-            driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist)
+            driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod,
+                                  *driver.arglist, coverage=True)
 
 
 def add_pythoneval(driver: Driver) -> None:
     driver.add_python_mod('eval-test', 'mypy.myunit',
-                          '-m', 'mypy.test.testpythoneval', *driver.arglist)
+                          '-m', 'mypy.test.testpythoneval', *driver.arglist,
+                         coverage=True)
 
 
 def add_cmdline(driver: Driver) -> None:
     driver.add_python_mod('cmdline-test', 'mypy.myunit',
-                          '-m', 'mypy.test.testcmdline', *driver.arglist)
+                          '-m', 'mypy.test.testcmdline', *driver.arglist,
+                         coverage=True)
 
 
 def add_stubs(driver: Driver) -> None:
@@ -288,6 +300,7 @@ def usage(status: int) -> None:
     print('  -l, --list             list included tasks (after filtering) and exit')
     print('  FILTER                 include tasks matching FILTER')
     print('  -x, --exclude FILTER   exclude tasks matching FILTER')
+    print('  -c, --coverage         calculate code coverage while running tests')
     print('  --                     treat all remaining arguments as positional')
     sys.exit(status)
 
@@ -316,6 +329,7 @@ def main() -> None:
     blacklist = []  # type: List[str]
     arglist = []  # type: List[str]
     list_only = False
+    coverage = False
 
     allow_opts = True
     curlist = whitelist
@@ -340,6 +354,8 @@ def main() -> None:
                 curlist = arglist
             elif a == '-l' or a == '--list':
                 list_only = True
+            elif a == '-c' or a == '--coverage':
+                coverage = True
             elif a == '-h' or a == '--help':
                 usage(0)
             else:
@@ -356,7 +372,7 @@ def main() -> None:
         whitelist.append('')
 
     driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist,
-            verbosity=verbosity, parallel_limit=parallel_limit, xfail=[])
+            verbosity=verbosity, parallel_limit=parallel_limit, xfail=[], coverage=coverage)
 
     driver.prepend_path('PATH', [join(driver.cwd, 'scripts')])
     driver.prepend_path('MYPYPATH', [driver.cwd])

setup.cfg

@@ -11,3 +11,11 @@ exclude = mypy/codec/*
 #   E704: multiple statements on one line (def)
 #   E402: module level import not at top of file
 ignore = E251,E128,F401,W601,E701,W503,E704,E402
+
+[coverage:run]
+branch = true
+source = mypy
+parallel = true
+
+[coverage:report]
+show_missing = true

test-requirements.txt

@@ -2,3 +2,4 @@ flake8
 typed-ast
 pytest>=2.8
 pytest-xdist>=1.13
+pytest-cov>=2.4.0