Make tests faster

lib/matplotlib/testing/compare.py

@@ -8,8 +8,11 @@
 
 import matplotlib
 from matplotlib.testing.noseclasses import ImageComparisonFailure
-from matplotlib.testing import image_util
+from matplotlib.testing import image_util, util
 from matplotlib import _png
+from matplotlib import _get_configdir
+from distutils import version
+import hashlib
 import math
 import operator
 import os
@@ -28,6 +31,15 @@
           ]
 
 #-----------------------------------------------------------------------
+
+def make_test_filename(fname, purpose):
+    """
+    Make a new filename by inserting `purpose` before the file's
+    extension.
+    """
+    base, ext = os.path.splitext(fname)
+    return '%s-%s%s' % (base, purpose, ext)
+
 def compare_float( expected, actual, relTol = None, absTol = None ):
    """Fail if the floating point values are not close enough, with
       the givem message.
@@ -87,35 +99,68 @@ def compare_float( expected, actual, relTol = None, absTol = None ):
 # A dictionary that maps filename extensions to functions that map
 # parameters old and new to a list that can be passed to Popen to
 # convert files with that extension to png format.
+def get_cache_dir():
+   cache_dir = os.path.join(_get_configdir(), 'test_cache')
+   if not os.path.exists(cache_dir):
+      try:
+         os.makedirs(cache_dir)
+      except IOError:
+         return None
+   if not os.access(cache_dir, os.W_OK):
+      return None
+   return cache_dir
+
+def get_file_hash(path, block_size=2**20):
+   md5 = hashlib.md5()
+   with open(path, 'rb') as fd:
+      while True:
+         data = fd.read(block_size)
+         if not data:
+            break
+         md5.update(data)
+   return md5.hexdigest()
+
 converter = { }
 
 def make_external_conversion_command(cmd):
-   def convert(*args):
-      cmdline = cmd(*args)
-      oldname, newname = args
+   def convert(old, new):
+      cmdline = cmd(old, new)
       pipe = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
       stdout, stderr = pipe.communicate()
       errcode = pipe.wait()
-      if not os.path.exists(newname) or errcode:
+      if not os.path.exists(new) or errcode:
          msg = "Conversion command failed:\n%s\n" % ' '.join(cmdline)
          if stdout:
             msg += "Standard output:\n%s\n" % stdout
          if stderr:
             msg += "Standard error:\n%s\n" % stderr
          raise IOError(msg)
+
    return convert
 
 if matplotlib.checkdep_ghostscript() is not None:
-   # FIXME: make checkdep_ghostscript return the command
-   if sys.platform == 'win32':
-      gs = 'gswin32c'
-   else:
-      gs = 'gs'
-   cmd = lambda old, new: \
-       [gs, '-q', '-sDEVICE=png16m', '-dNOPAUSE', '-dBATCH',
-        '-sOutputFile=' + new, old]
-   converter['pdf'] = make_external_conversion_command(cmd)
-   converter['eps'] = make_external_conversion_command(cmd)
+    def make_ghostscript_conversion_command():
+        # FIXME: make checkdep_ghostscript return the command
+        if sys.platform == 'win32':
+            gs = 'gswin32c'
+        else:
+            gs = 'gs'
+        cmd = [gs, '-q', '-sDEVICE=png16m', '-sOutputFile=-']
+
+        process = util.MiniExpect(cmd)
+
+        def do_convert(old, new):
+            process.expect("GS>")
+            process.sendline("(%s) run" % old)
+            with open(new, 'wb') as fd:
+                process.expect(">>showpage, press <return> to continue<<", fd)
+            process.sendline('')
+
+        return do_convert
+
+    converter['pdf'] = make_ghostscript_conversion_command()
+    converter['eps'] = make_ghostscript_conversion_command()
+
 
 if matplotlib.checkdep_inkscape() is not None:
    cmd = lambda old, new: \
@@ -127,7 +172,7 @@ def comparable_formats():
    on this system.'''
    return ['png'] + converter.keys()
 
-def convert(filename):
+def convert(filename, cache):
    '''
    Convert the named file into a png file.
    Returns the name of the created file.
@@ -138,11 +183,29 @@ def convert(filename):
    newname = base + '_' + extension + '.png'
    if not os.path.exists(filename):
       raise IOError("'%s' does not exist" % filename)
+
    # Only convert the file if the destination doesn't already exist or
    # is out of date.
    if (not os.path.exists(newname) or
        os.stat(newname).st_mtime < os.stat(filename).st_mtime):
+      if cache:
+         cache_dir = get_cache_dir()
+      else:
+         cache_dir = None
+
+      if cache_dir is not None:
+         hash = get_file_hash(filename)
+         new_ext = os.path.splitext(newname)[1]
+         cached_file = os.path.join(cache_dir, hash + new_ext)
+         if os.path.exists(cached_file):
+            shutil.copyfile(cached_file, newname)
+            return newname
+
       converter[extension](filename, newname)
+
+      if cache_dir is not None:
+         shutil.copyfile(newname, cached_file)
+
    return newname
 
 verifiers = { }
@@ -206,8 +269,8 @@ def compare_images( expected, actual, tol, in_decorator=False ):
    # Convert the image to png
    extension = expected.split('.')[-1]
    if extension != 'png':
-      actual = convert(actual)
-      expected = convert(expected)
+      actual = convert(actual, False)
+      expected = convert(expected, True)
 
    # open the image files and remove the alpha channel (if it exists)
    expectedImage = _png.read_png_int( expected )
@@ -216,24 +279,42 @@ def compare_images( expected, actual, tol, in_decorator=False ):
    actualImage, expectedImage = crop_to_same(actual, actualImage, expected, expectedImage)
 
    # normalize the images
-   expectedImage = image_util.autocontrast( expectedImage, 2 )
-   actualImage = image_util.autocontrast( actualImage, 2 )
+   # expectedImage = image_util.autocontrast( expectedImage, 2 )
+   # actualImage = image_util.autocontrast( actualImage, 2 )
 
    # compare the resulting image histogram functions
-   rms = 0
-   bins = np.arange(257)
-   for i in xrange(0, 3):
-      h1p = expectedImage[:,:,i]
-      h2p = actualImage[:,:,i]
+   expected_version = version.LooseVersion("1.6")
+   found_version = version.LooseVersion(np.__version__)
+
+   # On Numpy 1.6, we can use bincount with minlength, which is much faster than
+   # using histogram
+   if found_version >= expected_version:
+      rms = 0
+
+      for i in xrange(0, 3):
+         h1p = expectedImage[:,:,i]
+         h2p = actualImage[:,:,i]
+
+         h1h = np.bincount(h1p.ravel(), minlength=256)
+         h2h = np.bincount(h2p.ravel(), minlength=256)
+
+         rms += np.sum(np.power((h1h-h2h), 2))
+   else:
+      rms = 0
+      ns = np.arange(257)
+
+      for i in xrange(0, 3):
+         h1p = expectedImage[:,:,i]
+         h2p = actualImage[:,:,i]
+
+         h1h = np.histogram(h1p, bins=bins)[0]
+         h2h = np.histogram(h2p, bins=bins)[0]
 
-      h1h = np.histogram(h1p, bins=bins)[0]
-      h2h = np.histogram(h2p, bins=bins)[0]
+         rms += np.sum(np.power((h1h-h2h), 2))
 
-      rms += np.sum(np.power((h1h-h2h), 2))
    rms = np.sqrt(rms / (256 * 3))
 
-   diff_image = os.path.join(os.path.dirname(actual),
-                             'failed-diff-'+os.path.basename(actual))
+   diff_image = make_test_filename(actual, 'failed-diff')
 
    if ( (rms / 10000.0) <= tol ):
       if os.path.exists(diff_image):

lib/matplotlib/testing/decorators.py

@@ -6,10 +6,12 @@
 import matplotlib
 import matplotlib.tests
 import matplotlib.units
+from matplotlib import ticker
 from matplotlib import pyplot as plt
 from matplotlib import ft2font
 import numpy as np
-from matplotlib.testing.compare import comparable_formats, compare_images
+from matplotlib.testing.compare import comparable_formats, compare_images, \
+     make_test_filename
 import warnings
 
 def knownfailureif(fail_condition, msg=None, known_exception_class=None ):
@@ -98,6 +100,16 @@ def setup_class(cls):
 
         cls._func()
 
+    @staticmethod
+    def remove_text(figure):
+        figure.suptitle("")
+        for ax in figure.get_axes():
+            ax.set_title("")
+            ax.xaxis.set_major_formatter(ticker.NullFormatter())
+            ax.xaxis.set_minor_formatter(ticker.NullFormatter())
+            ax.yaxis.set_major_formatter(ticker.NullFormatter())
+            ax.yaxis.set_minor_formatter(ticker.NullFormatter())
+
     def test(self):
         baseline_dir, result_dir = _image_directories(self._func)
 
@@ -114,7 +126,8 @@ def test(self):
                 orig_expected_fname = os.path.join(baseline_dir, baseline) + '.' + extension
                 if extension == 'eps' and not os.path.exists(orig_expected_fname):
                     orig_expected_fname = os.path.join(baseline_dir, baseline) + '.pdf'
-                expected_fname = os.path.join(result_dir, 'expected-' + os.path.basename(orig_expected_fname))
+                expected_fname = make_test_filename(os.path.join(
+                    result_dir, os.path.basename(orig_expected_fname)), 'expected')
                 actual_fname = os.path.join(result_dir, baseline) + '.' + extension
                 if os.path.exists(orig_expected_fname):
                     shutil.copyfile(orig_expected_fname, expected_fname)
@@ -126,9 +139,13 @@ def test(self):
                     will_fail, fail_msg,
                     known_exception_class=ImageComparisonFailure)
                 def do_test():
+                    if self._remove_text:
+                        self.remove_text(figure)
+
                     figure.savefig(actual_fname)
 
-                    err = compare_images(expected_fname, actual_fname, self._tol, in_decorator=True)
+                    err = compare_images(expected_fname, actual_fname,
+                                         self._tol, in_decorator=True)
 
                     try:
                         if not os.path.exists(expected_fname):
@@ -148,7 +165,8 @@ def do_test():
 
                 yield (do_test,)
 
-def image_comparison(baseline_images=None, extensions=None, tol=1e-3, freetype_version=None):
+def image_comparison(baseline_images=None, extensions=None, tol=1e-3,
+                     freetype_version=None, remove_text=False):
     """
     call signature::
 
@@ -176,6 +194,11 @@ def image_comparison(baseline_images=None, extensions=None, tol=1e-3, freetype_v
       *freetype_version*: str or tuple
         The expected freetype version or range of versions for this
         test to pass.
+
+      *remove_text*: bool
+        Remove the title and tick text from the figure before
+        comparison.  This does not remove other, more deliberate,
+        text, such as legends and annotations.
     """
 
     if baseline_images is None:
@@ -207,7 +230,8 @@ def compare_images_decorator(func):
              '_baseline_images': baseline_images,
              '_extensions': extensions,
              '_tol': tol,
-             '_freetype_version': freetype_version})
+             '_freetype_version': freetype_version,
+             '_remove_text': remove_text})
 
         return new_class
     return compare_images_decorator
@@ -239,4 +263,3 @@ def _image_directories(func):
         os.makedirs(result_dir)
 
     return baseline_dir, result_dir
-

lib/matplotlib/testing/util.py

@@ -0,0 +1,67 @@
+import subprocess
+
+
+class MiniExpect:
+    """
+    This is a very basic version of pexpect, providing only the
+    functionality necessary for the testing framework, built on top of
+    `subprocess` rather than directly on lower-level calls.
+    """
+    def __init__(self, args):
+        """
+        Start the subprocess so it may start accepting commands.
+
+        *args* is a list of commandline arguments to pass to
+        `subprocess.Popen`.
+        """
+        self._name = args[0]
+        self._process = subprocess.Popen(
+            args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT)
+
+    def check_alive(self):
+        """
+        Raises a RuntimeError if the process is no longer alive.
+        """
+        returncode = self._process.poll()
+        if returncode is not None:
+            raise RuntimeError("%s unexpectedly quit" % self._name)
+
+    def sendline(self, line):
+        """
+        Send a line to the process.
+        """
+        self.check_alive()
+        stdin = self._process.stdin
+        stdin.write(line)
+        stdin.write('\n')
+        stdin.flush()
+
+    def expect(self, s, output=None):
+        """
+        Wait for the string *s* to appear in the child process's output.
+
+        *output* (optional) is a writable file object where all of the
+        content preceding *s* will be written.
+        """
+        self.check_alive()
+        read = self._process.stdout.read
+        pos = 0
+        buf = ''
+        while True:
+            char = read(1)
+            if not char:
+                raise IOError("Unexpected end-of-file")
+            elif char == s[pos]:
+                buf += char
+                pos += 1
+                if pos == len(s):
+                    return
+            else:
+                if output is not None:
+                    output.write(buf)
+                    output.write(char)
+                buf = ''
+                pos = 0