pytest-xdist + pytest-cov working

.travis/test.sh

@@ -1,17 +1,23 @@
 #!/bin/bash
 set -ex
-TOP=${TOP:-.}
+TOP=$(readlink -f ${TOP:-.})
 SITEPACKAGES=${SITEPACKAGES:-.}
 PROJECT=venv_update
+NCPU=$(getconf _NPROCESSORS_CONF)
 
 export PYTHONPATH=$(readlink -f $SITEPACKAGES)
 
 python --version
 coverage --version
 py.test --version
 coverage erase
-coverage run --parallel-mode --rcfile=$TOP/.coveragerc \
-    -m pytest "$@" $TOP/test $SITEPACKAGES/${PROJECT}.py
+py.test "$@" -n $NCPU \
+    --cov-enable --cov-config=$TOP/.coveragerc --cov-report='' \
+    $TOP/test $SITEPACKAGES/${PROJECT}.py
 coverage combine
 coverage report --fail-under 81  # FIXME: should be 100
-mv .coverage $TOP
+
+
+if [[ $CI ]]; then
+    mv .coverage $TOP
+fi

TODO

@@ -0,0 +1,9 @@
+# Things that I want to do, but would put me past my deadline:
+
+* I could remove the --cov-config with a small patch to pytest-cov
+    use os.path.abspath(options.cov_config) in postprocess(options)
+
+* coverage.py adds some helpful warnings to stderr, with no way to quiet them.
+    there's already an issue (#2 i think?), just needs a patch
+
+* pytest-timeout won't actually time-out with floating-point, zero, or negative input

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = -rfE --doctest-modules
+addopts = -v -rfE --doctest-modules
 norecursedirs =
     .tox
     tmp*

requirements.d/test.txt

@@ -1,4 +1,18 @@
-# minimal set of packages to run tests, under tox, for example
+# minimal set of packages to run tests (under tox, for example)
 -r testing.txt
+
+pytest-xdist
+pytest-timeout
+
+# NOTE: versions must be specified in dependency DFS post-order or else we get the wrong versions.
+
 #coverage  ## pending pull request: https://bitbucket.org/ned/coveragepy/pull-request/42
 hg+https://bitbucket.org/bukzor/coverage.py@__main__-support
+
+# cov-core ## pending pull requests:
+#       * https://github.com/schlamar/cov-core/pull/6
+#       * https://github.com/schlamar/cov-core/pull/7
+git+git://github.com/bukzor/cov-core.git@master
+
+# pytest-cov ## pending pull request: https://github.com/schlamar/pytest-cov/pull/21
+git+git://github.com/bukzor/pytest-cov.git@enable-coveragerc

requirements.d/testing.txt

@@ -1,5 +1,4 @@
-# packages imported by tests
+# minimal set of requirements to import (but not run) test code
+# this is used by the lint environment.
 -r prod.txt
-clom>=0.8.0a1
 pytest
-pytest-timeout

test/functional/simple_test.py

@@ -26,36 +26,26 @@ def run(cmd, *args, **env):
     check_call(cmd, env=env)
 
 
-def do_install(pwd, tmpdir, *args):
+def do_install(tmpdir, *args):
+    # we get coverage for free via the (patched) pytest-cov plugin
     run(
-        (
-            'coverage',
-            'run',
-            '--parallel-mode',
-            '--rcfile', str(TOP/'.coveragerc'),
-            '-m', 'venv_update',
-        ) + args,
-        COVERAGE_FILE=str(pwd/'.coverage'),
+        ('venv-update',) + args,
         HOME=str(tmpdir),
     )
 
 
 def test_trivial(tmpdir):
-    pwd = Path('.').realpath()
     tmpdir.chdir()
 
     # Trailing slash is essential to rsync
     run(('rsync', '-a', str(SCENARIOS) + '/trivial/', '.'))
-    do_install(pwd, tmpdir)
-
-    pwd.chdir()
+    do_install(tmpdir)
 
 
 # Not yet installed: https://github.com/klrmn/pytest-rerunfailures
 @pytest.mark.flaky(reruns=10)
 def test_second_install_faster(tmpdir):
     """install twice, and the second one should be faster, due to whl caching"""
-    pwd = Path('.').realpath()
     tmpdir.chdir()
 
     # Trailing slash is essential to rsync
@@ -67,15 +57,13 @@ def test_second_install_faster(tmpdir):
 
     from time import time
     start = time()
-    do_install(pwd, tmpdir)
+    do_install(tmpdir)
     time1 = time() - start
 
     start = time()
-    do_install(pwd, tmpdir)
+    do_install(tmpdir)
     time2 = time() - start
 
-    pwd.chdir()
-
     # second install should be at least twice as fast
     ratio = time1 / time2
     print('%.1fx speedup' % ratio)

test/testing/capture_subprocess_test.py

@@ -15,9 +15,9 @@ def make_outputter():
 seed(0)  # unpredictable, but repeatable
 
 # system should not deadlock for any given value of these parameters.
-LINES = 10
+LINES = 4000
 TIME = 0
-WIDTH = 79
+WIDTH = 179
 ERROR_RATIO = .20
 
 for i in range(LINES):
@@ -35,23 +35,34 @@ def make_outputter():
 ''')
 
 
+# coverage.py adds some helpful warnings to stderr, with no way to quiet them.
+from re import compile as Regex, MULTILINE
+coverage_warnings_regex = Regex(
+    r'^Coverage.py warning: (Module .* was never imported\.|No data was collected\.)\n',
+    flags=MULTILINE,
+)
+
+
 # buglet: floating-point, zero, negative values interpreted as infinite =/
-@pytest.mark.timeout(1)  # should take <.5s
+@pytest.mark.timeout(30)  # should take <2s
 def test_capture_subprocess(tmpdir):
     tmpdir.chdir()
     make_outputter()
 
     cmd = ('python', 'outputter.py')
     stdout, stderr, combined = capture_subprocess(cmd)
 
-    #assert stdout.count('\n') == 10
-    #assert stderr.count('\n') == 0
-    #assert combined.count('\n') == 14
+    stderr = coverage_warnings_regex.sub('', stderr)
+    combined = coverage_warnings_regex.sub('', combined)
+
+    assert stdout.count('\n') == 3207
+    assert stderr.count('\n') == 793
+    assert combined.count('\n') == 4000
 
-    #assert stdout.strip('.\r\n') == ''
-    #assert stderr.strip('%\r\n') == ''
+    assert stdout.strip('.\r\n') == ''
+    assert stderr.strip('%\r\n') == ''
 
-    # I'd like to assert that the two streams are interleaved strictly by line,
+    # I'd like to also assert that the two streams are interleaved strictly by line,
     # but I haven't been able to produce such output reliably =/
 
-    assert len(stdout) + len(stderr) == len(combined)
+    assert sorted(stdout + stderr) == sorted(combined)