Merge pull request #17 from cdent/skip-or-fail-feature

Add support for skipping or expecting failure on a test

docs/source/format.rst

@@ -61,6 +61,9 @@ these allow substitutions (explained below).
   in the response body.
 * ``response_json_paths``: A dictionary of JSONPath rules paired with
   expected matches.
+* ``skip``: A string message which if set will cause the test to be
+  skipped with the provided message.
+* ``xfail``: If ``True`` expect this test to fail but run it anyway.
 
 There are a small number of magical variables that can be used to make
 reference to the state of a current test or the one just prior. These

gabbi/case.py

@@ -13,23 +13,40 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-"""A single HTTP request reprensented as a ``unittest.TestCase``
+"""A single HTTP request represented as a subclass of ``unittest.TestCase``
 
 The test case encapsulates the request headers and body and expected
 response headers and body. When the test is run an HTTP request is
 made using httplib2. Assertions are made against the reponse.
 """
 
+import functools
 import json
 import os
 import re
+import sys
 
 import jsonpath_rw
 from six.moves.urllib import parse as urlparse
-import unittest
+from testtools import testcase
 
 
-class HTTPTestCase(unittest.TestCase):
+def potentialFailure(func):
+    """Decorate a test method that is expected to fail if 'xfail' is true."""
+    @functools.wraps(func)
+    def wrapper(self):
+        if self.test_data['xfail']:
+            try:
+                func(self)
+            except Exception:
+                raise testcase._ExpectedFailure(sys.exc_info())
+            raise testcase._UnexpectedSuccess
+        else:
+            func(self)
+    return wrapper
+
+
+class HTTPTestCase(testcase.TestCase):
     """Encapsulate a single HTTP request as a TestCase.
 
     If the test is a member of a sequence of requests, ensure that prior
@@ -48,13 +65,18 @@ def tearDown(self):
             super(HTTPTestCase, self).tearDown()
         self.has_run = True
 
+    @potentialFailure
     def test_request(self):
         """Run this request if it has not yet run.
 
         If there is a prior test in the sequence, run it first.
         """
         if self.has_run:
             return
+
+        if self.test_data['skip']:
+            self.skipTest(self.test_data['skip'])
+
         if self.prior and not self.prior.has_run:
             self.prior.run()
         self._run_test()

gabbi/driver.py

@@ -54,12 +54,13 @@
     'response_strings': None,
     'response_json_paths': None,
     'data': '',
+    'xfail': False,
+    'skip': '',
 }
 
 
 class TestBuilder(type):
-    """Metaclass to munge a dynamically created test.
-    """
+    """Metaclass to munge a dynamically created test."""
 
     required_attributes = {'has_run': False}
 
@@ -141,7 +142,7 @@ def test_suite_from_yaml(loader, test_base_name, test_yaml, test_directory,
                                  % test_name)
 
         # Use metaclasses to build a class of the necessary type
-        # with relevant arguments.
+        # and name with relevant arguments.
         klass = TestBuilder(test_name, (case.HTTPTestCase,),
                             {'test_data': test,
                              'test_directory': test_directory,

gabbi/gabbits_intercept/failskip.yaml

@@ -0,0 +1,14 @@
+#
+# Tests to confirm that xfail and skip are working.
+#
+
+tests:
+
+    - name: wrong status
+      xfail: True
+      url: /
+      status: 404
+
+    - name: skip me
+      skip: Skipping for now because we can't do it
+      url: http://nowhere.example.com/house

requirements.txt

@@ -1,5 +1,6 @@
 pbr>=0.6,!=0.7,<1.0
 six>=1.7.0
+testtools
 PyYAML
 httplib2
 jsonpath-rw

test-failskip.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Run the tests and confirm that the stuff we expect to skip or fail
+# does.
+
+GREP_FAIL_MATCH='expected failures=1'
+GREP_SKIP_MATCH='skips=1'
+
+python setup.py testr && \
+    testr last --subunit | subunit2pyunit 2>&1 | \
+    grep "${GREP_FAIL_MATCH}" && \
+    testr last | grep "${GREP_SKIP_MATCH}"

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = py27,py33,py34,pep8,limit
+envlist = py27,py33,py34,pep8,limit,failskip
 
 [testenv]
 deps = -r{toxinidir}/requirements.txt
@@ -23,6 +23,9 @@ commands =
 [testenv:limit]
 commands = {toxinidir}/test-limit.sh
 
+[testenv:failskip]
+commands = {toxinidir}/test-failskip.sh
+
 [testenv:cover]
 commands = python setup.py testr --coverage --testr-args="{posargs}"