Merge pull request #229 from cdent/bug/228

Fix pytest handling of fixture-level skips

.travis.yml

@@ -11,8 +11,6 @@ matrix:
         - env: TOXENV=py27-pytest
         - env: TOXENV=gnocchi
         - env: TOXENV=placement
-        - python: 3.3
-          env: TOXENV=py33
         - python: 3.4
           env: TOXENV=py34
         - python: pypy

gabbi/driver.py

@@ -152,7 +152,7 @@ def py_test_generator(test_dir, host=None, port=8001, intercept=None,
                       fixture_module=None, response_handlers=None,
                       content_handlers=None, require_ssl=False, url=None,
                       metafunc=None, use_prior_test=True,
-                      safe_yaml=True):
+                      inner_fixtures=None, safe_yaml=True):
     """Generate tests cases for py.test
 
     This uses build_tests to create TestCases and then yields them in

gabbi/pytester.py

@@ -90,14 +90,18 @@ def a_pytest_collection_modifyitems(items, config):
         if cleanname.startswith('start_'):
             test = item.callspec.params['test']
             result = item.callspec.params['result']
-            STARTS[suitename] = (test, result)
+            # TODO(cdent): Consider a named tuple here
+            STARTS[suitename] = (test, result, [])
             deselected.append(item)
         elif cleanname.startswith('stop_'):
             test = item.callspec.params['test']
             STOPS[suitename] = test
             deselected.append(item)
         else:
             remaining.append(item)
+            # Add each kept test to the start fixture
+            # in case we need to skip all the tests.
+            STARTS[suitename][2].append(item)
 
     if deselected:
         items[:] = remaining
@@ -123,8 +127,8 @@ def pytest_runtest_setup(item):
     run its priors after running this.
     """
     if hasattr(item, 'starter'):
-        test, result = item.starter
-        test(result)
+        test, result, tests = item.starter
+        test(result, tests)
 
 
 def pytest_runtest_teardown(item, nextitem):

gabbi/suite.py

@@ -82,7 +82,7 @@ def run(self, result, debug=False):
 
         return result
 
-    def start(self, result):
+    def start(self, result, tests):
         """Start fixtures when using pytest."""
         fixtures, intercept, host, port, prefix = self._get_intercept()
 
@@ -95,9 +95,9 @@ def start(self, result):
         except unittest.SkipTest as exc:
             # Disable the already collected tests that we now wish
             # to skip.
-            for test in self:
+            for test in tests:
                 test.run = noop
-                result.addSkip(test, str(exc))
+                test.add_marker('skip')
             result.addSkip(self, str(exc))
         if intercept:
             intercept_fixture = interceptor.Urllib3Interceptor(

gabbi/tests/gabbits_intercept/fixture.yaml

@@ -1,7 +1,7 @@
 
 fixtures:
-    - TestFixtureOne
-    - TestFixtureTwo
+    - FixtureOne
+    - FixtureTwo
 
 tests:
     - name: just to see

gabbi/tests/test_inner_fixture.py

@@ -21,6 +21,10 @@
 import fixtures
 
 from gabbi import driver
+# TODO(cdent): test_pytest allows pytest to see the tests this module
+# produces. Without it, the generator will not run. It is a todo because
+# needing to do this is annoying and gross.
+from gabbi.driver import test_pytest  # noqa
 from gabbi import fixture
 from gabbi.tests import simple_wsgi
 
@@ -54,10 +58,21 @@ def cleanUp(self):
         assert 1 <= COUNT_INNER <= 3
 
 
+BUILD_TEST_ARGS = dict(
+    intercept=simple_wsgi.SimpleWsgi,
+    fixture_module=sys.modules[__name__],
+    inner_fixtures=[InnerFixture],
+)
+
+
 def load_tests(loader, tests, pattern):
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return driver.build_tests(test_dir, loader, host=None,
-                              intercept=simple_wsgi.SimpleWsgi,
-                              fixture_module=sys.modules[__name__],
-                              inner_fixtures=[InnerFixture],
-                              test_loader_name=__name__)
+    return driver.build_tests(test_dir, loader,
+                              test_loader_name=__name__,
+                              **BUILD_TEST_ARGS)
+
+
+def pytest_generate_tests(metafunc):
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    driver.py_test_generator(test_dir, metafunc=metafunc,
+                             **BUILD_TEST_ARGS)

gabbi/tests/test_intercept.py

@@ -20,6 +20,10 @@
 import sys
 
 from gabbi import driver
+# TODO(cdent): test_pytest allows pytest to see the tests this module
+# produces. Without it, the generator will not run. It is a todo because
+# needing to do this is annoying and gross.
+from gabbi.driver import test_pytest  # noqa
 from gabbi import fixture
 from gabbi.handlers import base
 from gabbi.tests import simple_wsgi
@@ -29,17 +33,17 @@
 TESTS_DIR = 'gabbits_intercept'
 
 
-class TestFixtureOne(fixture.GabbiFixture):
+class FixtureOne(fixture.GabbiFixture):
     """Drive the fixture testing weakly."""
     pass
 
 
-class TestFixtureTwo(fixture.GabbiFixture):
+class FixtureTwo(fixture.GabbiFixture):
     """Drive the fixture testing weakly."""
     pass
 
 
-class TestResponseHandler(base.ResponseHandler):
+class StubResponseHandler(base.ResponseHandler):
     """A sample response handler just to test."""
 
     test_key_suffix = 'test'
@@ -62,16 +66,27 @@ def action(self, test, expected, value=None):
 SkipAllFixture = fixture.SkipAllFixture
 
 
+BUILD_TEST_ARGS = dict(
+    intercept=simple_wsgi.SimpleWsgi,
+    fixture_module=sys.modules[__name__],
+    prefix=os.environ.get('GABBI_PREFIX'),
+    response_handlers=[StubResponseHandler]
+)
+
+
 def load_tests(loader, tests, pattern):
     """Provide a TestSuite to the discovery process."""
     # Set and environment variable for one of the tests.
     util.set_test_environ()
 
-    prefix = os.environ.get('GABBI_PREFIX')
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return driver.build_tests(test_dir, loader, host=None,
-                              intercept=simple_wsgi.SimpleWsgi,
+    return driver.build_tests(test_dir, loader,
                               test_loader_name=__name__,
-                              prefix=prefix,
-                              fixture_module=sys.modules[__name__],
-                              response_handlers=[TestResponseHandler])
+                              **BUILD_TEST_ARGS)
+
+
+def pytest_generate_tests(metafunc):
+    util.set_test_environ()
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    driver.py_test_generator(test_dir, metafunc=metafunc,
+                             **BUILD_TEST_ARGS)

gabbi/tests/test_live.py

@@ -19,6 +19,10 @@
 from unittest import case
 
 from gabbi import driver
+# TODO(cdent): test_pytest allows pytest to see the tests this module
+# produces. Without it, the generator will not run. It is a todo because
+# needing to do this is annoying and gross.
+from gabbi.driver import test_pytest  # noqa
 from gabbi import fixture
 
 
@@ -33,9 +37,19 @@ def start_fixture(self):
             raise case.SkipTest('live tests skipped')
 
 
+BUILD_TEST_ARGS = dict(
+    host='google.com',
+    fixture_module=sys.modules[__name__],
+    port=443
+)
+
+
 def load_tests(loader, tests, pattern):
     """Provide a TestSuite to the discovery process."""
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return driver.build_tests(test_dir, loader, host='google.com',
-                              fixture_module=sys.modules[__name__],
-                              port=443)
+    return driver.build_tests(test_dir, loader, **BUILD_TEST_ARGS)
+
+
+def pytest_generate_tests(metafunc):
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    driver.py_test_generator(test_dir, metafunc=metafunc, **BUILD_TEST_ARGS)

gabbi/tests/test_runner.py

@@ -180,7 +180,7 @@ def test_custom_response_handler(self):
                 self.assertFailure(err)
 
         sys.argv.insert(3, "-r")
-        sys.argv.insert(4, "gabbi.tests.test_intercept:TestResponseHandler")
+        sys.argv.insert(4, "gabbi.tests.test_intercept:StubResponseHandler")
 
         sys.stdin = StringIO("""
         tests:

gabbi/tests/test_unsafe_yaml.py

@@ -21,6 +21,10 @@
 import yaml
 
 from gabbi import driver
+# TODO(cdent): test_pytest allows pytest to see the tests this module
+# produces. Without it, the generator will not run. It is a todo because
+# needing to do this is annoying and gross.
+from gabbi.driver import test_pytest  # noqa
 from gabbi.tests import simple_wsgi
 from gabbi.tests import util
 
@@ -31,15 +35,20 @@
 yaml.add_constructor(u'!IsNAN', lambda loader, node: util.NanChecker())
 
 
+BUILD_TEST_ARGS = dict(
+    intercept=simple_wsgi.SimpleWsgi,
+    safe_yaml=False
+)
+
+
 def load_tests(loader, tests, pattern):
     """Provide a TestSuite to the discovery process."""
-    # Set and environment variable for one of the tests.
-    util.set_test_environ()
-
-    prefix = os.environ.get('GABBI_PREFIX')
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return driver.build_tests(test_dir, loader, host=None,
-                              intercept=simple_wsgi.SimpleWsgi,
+    return driver.build_tests(test_dir, loader,
                               test_loader_name=__name__,
-                              prefix=prefix,
-                              safe_yaml=False)
+                              **BUILD_TEST_ARGS)
+
+
+def pytest_generate_tests(metafunc):
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    driver.py_test_generator(test_dir, metafunc=metafunc, **BUILD_TEST_ARGS)

setup.cfg

@@ -16,7 +16,6 @@ classifier =
     Programming Language :: Python :: 2
     Programming Language :: Python :: 2.7
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.3
     Programming Language :: Python :: 3.4
     Programming Language :: Python :: 3.5
     Programming Language :: Python :: 3.6

test-failskip.sh

@@ -7,12 +7,14 @@ shopt -s nocasematch
 [[ "${GABBI_SKIP_NETWORK:-false}" == "true" ]] && SKIP=7 || SKIP=2
 shopt -u nocasematch
 
-GREP_FAIL_MATCH='expected failures=12,'
+FAILS=12
+
+GREP_FAIL_MATCH="expected failures=$FAILS,"
 GREP_SKIP_MATCH="skipped=$SKIP,"
 GREP_UXSUC_MATCH='unexpected successes=1'
 # This skip is always 2 because the pytest tests don't
 # run the live tests.
-PYTEST_MATCH='2 skipped, 12 xfailed'
+PYTEST_MATCH="$SKIP skipped, $FAILS xfailed"
 
 python setup.py testr && \
     for match in "${GREP_FAIL_MATCH}" "${GREP_UXSUC_MATCH}" "${GREP_SKIP_MATCH}"; do

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = py27,py33,py34,py35,py36,pypy,pep8,limit,failskip,docs,py35-prefix,py35-limit,py35-failskip,py27-pytest,py35-pytest,py36-pytest
+envlist = py27,py34,py35,py36,pypy,pep8,limit,failskip,docs,py35-prefix,py35-limit,py35-failskip,py27-pytest,py35-pytest,py36-pytest
 
 [testenv]
 deps = -r{toxinidir}/requirements.txt