Added an improved test_function_new decorator to pytest_extensions

The `pytest_extensions.test_function` decorator is signature-preserving.
This causes the decorated test functions to have unused arguments, and access
to the `kwargs` dict in the test function needs to by key access to the
`kwargs` dict, causing extra code in the test functions.

This change introduces a `pytest_extensions.test_function_new` decorator
with an improved interface for the test functions which now get the testcase
tuple as a named tuple, and the `kwargs` dict of the testcase as expanded
keyword arguments.

Note that the new decorator needs to explicitly set the __signature__
attribute of the test function, because of a specific pytest
default behavior of using the signature of the unpacked wrapped function.
See also pytest issue #3435.

Signed-off-by: Andreas Maier <maiera@de.ibm.com>

testsuite/pytest_extensions.py

@@ -5,10 +5,13 @@
 from __future__ import absolute_import
 
 import pytest
+import functools
+import inspect
+from collections import namedtuple
 from decorator import decorator
+import six
 
-
-__all__ = ['test_function']
+__all__ = ['test_function', 'test_function_new']
 
 
 @decorator
@@ -137,3 +140,166 @@ def test_CIMClass_equal(
 
             ret = None  # Debugging hint
     return ret
+
+
+def test_function_new(test_func):
+    """
+    A decorator for test functions that simplifies the test function by
+    handling a number of things:
+
+    * Skipping the test if the `condition` item in the testcase is `False`,
+    * Invoking the Python debugger if the `condition` item in the testcase is
+      the string "pdb",
+    * Capturing and validating any warnings issued by the test function,
+      if the `exp_warn_types` item in the testcase is set,
+    * Catching and validating any exceptions raised by the test function,
+      if the `exp_exc_types` item in the testcase is set.
+
+    This is a signature-changing decorator.
+
+    Parameters of the wrapper function:
+
+    * desc (string): Short testcase description.
+
+    * kwargs (dict): Keyword arguments for the test function.
+
+    * exp_exc_types (Exception or list of Exception): Expected exception types,
+      or `None` if no exceptions are expected.
+
+    * exp_warn_types (Warning or list of Warning): Expected warning types,
+      or `None` if no warnings are expected.
+
+    * condition (bool or 'pdb'): Boolean condition for running the testcase.
+      If it evaluates to `bool(False)`, the testcase will be skipped.
+      If it evaluates to `bool(True)`, the testcase will be run.
+      The string value 'pdb' will cause the Python pdb debugger to be entered
+      before calling the test function.
+
+    Parameters of the test function that is decorated:
+
+    * testcase (testcase_tuple): The testcase, as a named tuple.
+
+    * **kwargs: Keyword arguments for the test function.
+
+    Notes:
+
+    * Using this decorator together with the `pytest.mark.parametrize`
+      decorator requires applying this decorator first to the test function
+      (see the example).
+
+    Example::
+
+        TESTCASES_CIMCLASS_EQUAL = [
+            # desc, kwargs, exp_exc_types, exp_warn_types, condition
+            (
+                "Equality with different lexical case of name",
+                dict(
+                    obj1=CIMClass('CIM_Foo'),
+                    obj2=CIMClass('cim_foo'),
+                    exp_equal=True,
+                ),
+                None, None, True
+            ),
+            # ... more testcases
+        ]
+
+        @pytest.mark.parametrize(
+            "desc, kwargs, exp_exc_types, exp_warn_types, condition",
+            TESTCASES_CIMCLASS_EQUAL)
+        @pytest_extensions.test_function_new
+        def test_CIMClass_equal(testcase, obj1, obj2, exp_equal):
+
+            # The code to be tested
+            equal = (obj1 == obj2)
+
+            # Verify that an exception raised in this function is not mistaken
+            # to be the expected exception
+            assert testcase.exp_exc_types is None
+
+            # Verify the result
+            assert equal == exp_equal
+    """
+
+    # A testcase tuple
+    testcase_tuple = namedtuple(
+        'testcase_tuple',
+        ['desc', 'kwargs', 'exp_exc_types', 'exp_warn_types', 'condition']
+    )
+
+    def wrapper_func(desc, kwargs, exp_exc_types, exp_warn_types, condition):
+        """
+        Wrapper function that calls the test function that is decorated.
+        """
+
+        if not condition:
+            pytest.skip("Condition for test case not met")
+
+        if condition == 'pdb':
+            import pdb
+
+        testcase = testcase_tuple(desc, kwargs, exp_exc_types, exp_warn_types,
+                                  condition)
+
+        if exp_warn_types:
+            with pytest.warns(exp_warn_types) as rec_warnings:
+                if exp_exc_types:
+                    with pytest.raises(exp_exc_types):
+                        if condition == 'pdb':
+                            pdb.set_trace()
+
+                        test_func(testcase, **kwargs)  # expecting an exception
+
+                    ret = None  # Debugging hint
+                    # In combination with exceptions, we do not verify warnings
+                    # (they could have been issued before or after the
+                    # exception).
+                else:
+                    if condition == 'pdb':
+                        pdb.set_trace()
+
+                    test_func(testcase, **kwargs)  # not expecting an exception
+
+                    ret = None  # Debugging hint
+                    assert len(rec_warnings) >= 1
+        else:
+            if exp_exc_types:
+                with pytest.raises(exp_exc_types):
+                    if condition == 'pdb':
+                        pdb.set_trace()
+
+                    test_func(testcase, **kwargs)  # expecting an exception
+
+                ret = None  # Debugging hint
+            else:
+                if condition == 'pdb':
+                    pdb.set_trace()
+
+                test_func(testcase, **kwargs)  # not expecting an exception
+
+                ret = None  # Debugging hint
+        return ret
+
+    # Pytest determines the signature of the test function decorated with this
+    # decorator, from its __signature__ attribute, or if not set defaults to
+    # inspect.signature(..., follow_wrapped=True), which returns the
+    # (incorrect) signature of the wrapped (original) function. We correct this
+    # behavior by setting the __signature__ attribute of the wrapper function
+    # to its correct signature. To do that, we cannot use inspect.signature()
+    # because its follow_wrapped parameter was introduced only in Python 3.5.
+    if six.PY3:
+        wrapper_func.__signature__ = inspect.Signature(
+            parameters=[
+                inspect.Parameter('desc',
+                                  inspect.Parameter.POSITIONAL_OR_KEYWORD),
+                inspect.Parameter('kwargs',
+                                  inspect.Parameter.POSITIONAL_OR_KEYWORD),
+                inspect.Parameter('exp_exc_types',
+                                  inspect.Parameter.POSITIONAL_OR_KEYWORD),
+                inspect.Parameter('exp_warn_types',
+                                  inspect.Parameter.POSITIONAL_OR_KEYWORD),
+                inspect.Parameter('condition',
+                                  inspect.Parameter.POSITIONAL_OR_KEYWORD),
+            ]
+        )
+
+    return functools.update_wrapper(wrapper_func, test_func)