Prepare for cross-framework test suite

  - introduced `xfail(reason)` function
    - uses: `raise KnownFailureTest(msg)` -> `xfail(reason)`
    - same name and signature as in pytest

  - introduced `skip(reason)` function
    - uses: `raise SkipTest(msg)` -> `skip(reason)`
    - same name and signature as in pytest

  - introduced `skipif(condition, reason=None)` decorator
    - uses: replaces `def func(): if condition: skip()`
    - same name and signature as in pytest
    - can be used with functions, classes, and methods
    - supports string condition (evaluated at runtime)

  - moved nose related code to `testing.nose` submodule
    - plugins in `testing.nose.plugins` submodule
    - decorators implementation in `testing.nose.decorators`
      (interface is still in `testing.decorators`, implementation will
       have been chosen at runtime according to used test framework)

  - `matplotlib.test` function unifications
  - `tests.py` now uses `matplotlib.test()`

lib/matplotlib/__init__.py

@@ -1398,7 +1398,7 @@ def use(arg, warn=True, force=False):
     if 'matplotlib.backends' in sys.modules:
         # Warn only if called with a different name
         if (rcParams['backend'] != name) and warn:
-            warnings.warn(_use_error_msg)
+            warnings.warn(_use_error_msg, stacklevel=2)
 
         # Unless we've been told to force it, just return
         if not force:
@@ -1580,70 +1580,17 @@ def _init_tests():
             )
         )
 
-    try:
-        import nose
-        try:
-            from unittest import mock
-        except:
-            import mock
-    except ImportError:
-        print("matplotlib.test requires nose and mock to run.")
-        raise
-
-
-def _get_extra_test_plugins():
-    from .testing.performgc import PerformGC
-    from .testing.noseclasses import KnownFailure
-    from nose.plugins import attrib
+    from .testing.nose import check_deps
+    check_deps()
 
-    return [PerformGC, KnownFailure, attrib.Plugin]
 
-
-def _get_nose_env():
-    env = {'NOSE_COVER_PACKAGE': 'matplotlib',
-           'NOSE_COVER_HTML': 1,
-           'NOSE_COVER_NO_PRINT': 1}
-    return env
-
-
-def test(verbosity=1, coverage=False):
+def test(verbosity=1, coverage=False, **kwargs):
     """run the matplotlib test suite"""
     _init_tests()
 
-    old_backend = rcParams['backend']
-    try:
-        use('agg')
-        import nose
-        import nose.plugins.builtin
-        from nose.plugins.manager import PluginManager
-        from nose.plugins import multiprocess
-
-        # store the old values before overriding
-        plugins = _get_extra_test_plugins()
-        plugins.extend([plugin for plugin in nose.plugins.builtin.plugins])
-
-        manager = PluginManager(plugins=[x() for x in plugins])
-        config = nose.config.Config(verbosity=verbosity, plugins=manager)
-
-        # Nose doesn't automatically instantiate all of the plugins in the
-        # child processes, so we have to provide the multiprocess plugin with
-        # a list.
-        multiprocess._instantiate_plugins = plugins
-
-        env = _get_nose_env()
-        if coverage:
-            env['NOSE_WITH_COVERAGE'] = 1
-
-        success = nose.run(
-            defaultTest=default_test_modules,
-            config=config,
-            env=env,
-        )
-    finally:
-        if old_backend.lower() != 'agg':
-            use(old_backend)
+    from .testing.nose import test as nose_test
+    return nose_test(verbosity, coverage, **kwargs)
 
-    return success
 
 test.__test__ = False  # nose: this function is not a test
 

lib/matplotlib/testing/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
+import inspect
 import warnings
 from contextlib import contextmanager
 
@@ -13,6 +14,49 @@ def _is_list_like(obj):
     return not is_string_like(obj) and iterable(obj)
 
 
+def xfail(msg=""):
+    """Explicitly fail an currently-executing test with the given message."""
+    from .nose import knownfail
+    knownfail(msg)
+
+
+def skip(msg=""):
+    """Skip an executing test with the given message."""
+    from nose import SkipTest
+    raise SkipTest(msg)
+
+
+# stolen from pytest
+def getrawcode(obj, trycall=True):
+    """Return code object for given function."""
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+
+def copy_metadata(src_func, tgt_func):
+    """Replicates metadata of the function. Returns target function."""
+    tgt_func.__dict__ = src_func.__dict__
+    tgt_func.__doc__ = src_func.__doc__
+    tgt_func.__module__ = src_func.__module__
+    tgt_func.__name__ = src_func.__name__
+    if hasattr(src_func, '__qualname__'):
+        tgt_func.__qualname__ = src_func.__qualname__
+    if not hasattr(tgt_func, 'compat_co_firstlineno'):
+        tgt_func.compat_co_firstlineno = getrawcode(src_func).co_firstlineno
+    return tgt_func
+
+
 # stolen from pandas
 @contextmanager
 def assert_produces_warning(expected_warning=Warning, filter_level="always",

lib/matplotlib/testing/decorators.py

@@ -4,17 +4,13 @@
 import six
 
 import functools
-import gc
 import inspect
 import os
 import sys
 import shutil
 import warnings
 import unittest
 
-import nose
-import numpy as np
-
 import matplotlib as mpl
 import matplotlib.style
 import matplotlib.units
@@ -24,13 +20,23 @@
 from matplotlib import pyplot as plt
 from matplotlib import ft2font
 from matplotlib import rcParams
-from matplotlib.testing.noseclasses import KnownFailureTest, \
-     KnownFailureDidNotFailTest, ImageComparisonFailure
 from matplotlib.testing.compare import comparable_formats, compare_images, \
      make_test_filename
+from . import copy_metadata, skip, xfail
+from .exceptions import ImageComparisonFailure
+
+
+def skipif(condition, *args, **kwargs):
+    """Skip the given test function if eval(condition) results in a True
+    value.
+
+    Optionally specify a reason for better reporting.
+    """
+    from .nose.decorators import skipif
+    return skipif(condition, *args, **kwargs)
 
 
-def knownfailureif(fail_condition, msg=None, known_exception_class=None ):
+def knownfailureif(fail_condition, msg=None, known_exception_class=None):
     """
 
     Assume a will fail if *fail_condition* is True. *fail_condition*
@@ -42,32 +48,8 @@ def knownfailureif(fail_condition, msg=None, known_exception_class=None ):
     if the exception is an instance of this class. (Default = None)
 
     """
-    # based on numpy.testing.dec.knownfailureif
-    if msg is None:
-        msg = 'Test known to fail'
-    def known_fail_decorator(f):
-        # Local import to avoid a hard nose dependency and only incur the
-        # import time overhead at actual test-time.
-        import nose
-        def failer(*args, **kwargs):
-            try:
-                # Always run the test (to generate images).
-                result = f(*args, **kwargs)
-            except Exception as err:
-                if fail_condition:
-                    if known_exception_class is not None:
-                        if not isinstance(err,known_exception_class):
-                            # This is not the expected exception
-                            raise
-                    # (Keep the next ultra-long comment so in shows in console.)
-                    raise KnownFailureTest(msg) # An error here when running nose means that you don't have the matplotlib.testing.noseclasses:KnownFailure plugin in use.
-                else:
-                    raise
-            if fail_condition and fail_condition != 'indeterminate':
-                raise KnownFailureDidNotFailTest(msg)
-            return result
-        return nose.tools.make_decorator(f)(failer)
-    return known_fail_decorator
+    from .nose.decorators import knownfailureif
+    return knownfailureif(fail_condition, msg, known_exception_class)
 
 
 def _do_cleanup(original_units_registry, original_settings):
@@ -211,7 +193,7 @@ def remove_text(figure):
     def test(self):
         baseline_dir, result_dir = _image_directories(self._func)
         if self._style != 'classic':
-            raise KnownFailureTest('temporarily disabled until 2.0 tag')
+            xfail('temporarily disabled until 2.0 tag')
         for fignum, baseline in zip(plt.get_fignums(), self._baseline_images):
             for extension in self._extensions:
                 will_fail = not extension in comparable_formats()
@@ -263,13 +245,14 @@ def do_test():
                                 '(RMS %(rms).3f)'%err)
                     except ImageComparisonFailure:
                         if not check_freetype_version(self._freetype_version):
-                            raise KnownFailureTest(
+                            xfail(
                                 "Mismatched version of freetype.  Test requires '%s', you have '%s'" %
                                 (self._freetype_version, ft2font.__freetype_version__))
                         raise
 
                 yield (do_test,)
 
+
 def image_comparison(baseline_images=None, extensions=None, tol=0,
                      freetype_version=None, remove_text=False,
                      savefig_kwarg=None, style='classic'):
@@ -428,7 +411,7 @@ def backend_switcher(*args, **kwargs):
                 plt.switch_backend(prev_backend)
             return result
 
-        return nose.tools.make_decorator(func)(backend_switcher)
+        return copy_metadata(func, backend_switcher)
     return switch_backend_decorator
 
 
@@ -447,7 +430,6 @@ def skip_if_command_unavailable(cmd):
     try:
         check_output(cmd)
     except:
-        from nose import SkipTest
-        raise SkipTest('missing command: %s' % cmd[0])
+        skip('missing command: %s' % cmd[0])
 
     return lambda f: f

lib/matplotlib/testing/exceptions.py

@@ -1,15 +1,3 @@
-class KnownFailureTest(Exception):
-    """
-    Raise this exception to mark a test as a known failing test.
-    """
-
-
-class KnownFailureDidNotFailTest(Exception):
-    """
-    Raise this exception to mark a test should have failed but did not.
-    """
-
-
 class ImageComparisonFailure(AssertionError):
     """
     Raise this exception to mark a test as a comparison between two images.

lib/matplotlib/testing/nose/__init__.py

@@ -0,0 +1,70 @@
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+
+def get_extra_test_plugins():
+    from .plugins.performgc import PerformGC
+    from .plugins.knownfailure import KnownFailure
+    from nose.plugins import attrib
+
+    return [PerformGC, KnownFailure, attrib.Plugin]
+
+
+def get_env():
+    env = {'NOSE_COVER_PACKAGE': 'matplotlib',
+           'NOSE_COVER_HTML': 1,
+           'NOSE_COVER_NO_PRINT': 1}
+    return env
+
+
+def check_deps():
+    try:
+        import nose
+        try:
+            from unittest import mock
+        except ImportError:
+            import mock
+    except ImportError:
+        print("matplotlib.test requires nose and mock to run.")
+        raise
+
+
+def test(verbosity=None, coverage=False, switch_backend_warn=True, **kwargs):
+    from ... import default_test_modules, get_backend, use
+
+    old_backend = get_backend()
+    try:
+        use('agg')
+        import nose
+        from nose.plugins import multiprocess
+
+        # Nose doesn't automatically instantiate all of the plugins in the
+        # child processes, so we have to provide the multiprocess plugin with
+        # a list.
+        extra_plugins = get_extra_test_plugins()
+        multiprocess._instantiate_plugins = extra_plugins
+
+        env = get_env()
+        if coverage:
+            env['NOSE_WITH_COVERAGE'] = 1
+
+        if verbosity is not None:
+            env['NOSE_VERBOSE'] = verbosity
+
+        success = nose.run(
+            addplugins=[plugin() for plugin in extra_plugins],
+            env=env,
+            defaultTest=default_test_modules,
+            **kwargs
+        )
+    finally:
+        if old_backend.lower() != 'agg':
+            use(old_backend, warn=switch_backend_warn)
+
+    return success
+
+
+def knownfail(msg):
+    from .exceptions import KnownFailureTest
+    # Keep the next ultra-long comment so it shows in console.
+    raise KnownFailureTest(msg)  # An error here when running nose means that you don't have the matplotlib.testing.nose.plugins:KnownFailure plugin in use.  # noqa