Merge pull request #1277 from ayshih/hash

Hash-based figure verification for tests

CHANGELOG.md

@@ -18,6 +18,7 @@ Latest
  * `sunpy.cm.get_cmap` no longer defaults to 'sdoaia94'
  * Added database url config setting to be setup by default as a sqlite database in the sunpy working directory
  * Added a few tests for the sunpy.roi module
+ * Added capability for figure-based tests
  * Refactored mapcube co-alignment functionality.
  * Removed sample data from distribution and added ability to download sample files
  * Require JSOC request data calls have an email address attached.

MANIFEST.in

@@ -8,6 +8,7 @@ include RELEASE.md
 include CHANGELOG.md
 include licenses/*.rst
 include sunpy/data/sunpyrc
+include sunpy/tests/figure_hashes.json
 include setup.py
 include ah_bootstrap.py
 include ez_setup.py

doc/source/dev.rst

@@ -785,6 +785,37 @@ test is skipped, otherwise it is run. Marking tests is pretty
 straightforward in pytest: use the decorator ``@pytest.mark.online`` to
 mark a test function as needing an internet connection.
 
+Writing a unit test for a figure
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can write SunPy unit tests that test the generation of matplotlib figures
+by adding the decorator `sunpy.tests.helpers.figure_test`.
+Here is a simple example: ::
+
+    import matplotlib.pyplot as plt
+    from sunpy.tests.helpers import figure_test
+
+    @figure_test
+    def test_simple_plot():
+        plt.plot([0,1])
+
+The current figure at the end of the unit test, or an explicitly returned
+figure, has its hash compared against an established hash library (more on
+this below).  If the hashes do not match, the figure has changed, and thus
+the test is considered to have failed.
+
+All such tests are automatically marked with the pytest mark
+`pytest.mark.figure`.  See the next section for how to use marks.
+
+You will need to update the library of figure hashes after you create a new
+figure test or after a figure has intentionally changed due to code improvement.
+Once you have confirmed that the only figure-test failures are anticipated ones,
+remove the existing hash library (found at `sunpy/tests/figure_hashes.json`)
+and then run the entire suite of SunPy tests.  Note that all figure tests will
+fail since a new hash library needs to be built.  The test report will tell you
+where the new hash library has been created, which you then copy to
+`sunpy/tests/`.
+
 Running unit tests
 ^^^^^^^^^^^^^^^^^^
 
@@ -802,7 +833,7 @@ the module on the command line, e.g.::
 for the tests for `sunpy.util.xml`.
 
 To run only tests that been marked with a specific pytest mark using the
-decorator ``@pytest.mark`` (the the section *Writing a unit test*), use the
+decorator ``@pytest.mark`` (see the section *Writing a unit test*), use the
 following command (where ``MARK`` is the name of the mark)::
 
   py.test -k MARK

pytest.ini

@@ -2,3 +2,6 @@
 norecursedirs = build *.egg-info
 python_files = test_?*.py
 addopts = -p no:doctest
+markers =
+    online: marks this test function as needing online connectivity.
+    figure: marks this test function as using hash-based Matplotlib figure verification. This mark is not meant to be directly applied, but is instead automatically applied when a test function uses the @sunpy.tests.helpers.figure_test decorator.

sunpy/tests/conftest.py → sunpy/conftest.py

@@ -1,5 +1,8 @@
 from functools import partial
 import urllib2
+import os
+import tempfile
+import json
 
 import pytest
 
@@ -11,6 +14,10 @@
 else:
     matplotlib.use('Agg')
 
+from sunpy.tests import hash
+
+hash_library_original_len = len(hash.hash_library)
+
 GOOGLE_URL = 'http://www.google.com'
 
 
@@ -36,3 +43,14 @@ def pytest_runtest_setup(item):
         if 'online' in item.keywords and not is_online():
             msg = 'skipping test {0} (reason: client seems to be offline)'
             pytest.skip(msg.format(item.name))
+
+def pytest_unconfigure(config):
+    #Check if additions have been made to the hash library
+    if len(hash.hash_library) > hash_library_original_len:
+        #Write the new hash library in JSON
+        tempdir = tempfile.mkdtemp()
+        hashfile = os.path.join(tempdir, hash.HASH_LIBRARY_NAME)
+        with open(hashfile, 'wb') as outfile:
+            json.dump(hash.hash_library, outfile, sort_keys=True, indent=4, separators=(',', ': '))
+        print("The hash library has expanded and should be copied to sunpy/tests/")
+        print("  " + hashfile)

sunpy/map/tests/test_mapbase.py

@@ -20,6 +20,7 @@
 import sunpy.map
 import sunpy.data.test
 from sunpy.time import parse_time
+from sunpy.tests.helpers import figure_test
 
 testpath = sunpy.data.test.rootdir
 
@@ -406,3 +407,8 @@ def test_rotate_invalid_order(generic_map):
         generic_map.rotate(order=6)
     with pytest.raises(ValueError):
         generic_map.rotate(order=-1)
+
+
+@figure_test
+def test_plot_aia171(aia171_test_map):
+    aia171_test_map.plot()

sunpy/tests/figure_hashes.json

@@ -0,0 +1,3 @@
+{
+    "sunpy.map.tests.test_mapbase.test_plot_aia171": "62b5b9fc1a423e3cbeeea85d3b8daf1a7877099fdeda2ce9352746be1dd451fc"
+}

sunpy/tests/hash.py

@@ -0,0 +1,66 @@
+import os
+import io
+import hashlib
+import json
+
+import matplotlib.pyplot as plt
+
+HASH_LIBRARY_NAME = 'figure_hashes.json'
+
+# Load the hash library if it exists
+try:
+    with open(os.path.join(os.path.dirname(__file__), HASH_LIBRARY_NAME)) as infile:
+        hash_library = json.load(infile)
+except IOError:
+    hash_library = {}
+
+def hash_figure(figure=None):
+    """
+    For a matplotlib.figure.Figure, returns the SHA256 hash as a hexadecimal string.
+
+    Parameters
+    ----------
+    figure : matplotlib.figure.Figure
+        If None is specified, the current figure is used (as determined by matplotlib.pyplot.gcf())
+
+    Returns
+    -------
+    out : string
+        The SHA256 hash in hexadecimal representation
+    """
+
+    if figure is None:
+        figure = plt.gcf()
+
+    imgdata = io.BytesIO()
+    figure.savefig(imgdata, format='png')
+
+    imgdata.seek(0)
+    buf = imgdata.read()
+    imgdata.close()
+
+    hasher = hashlib.sha256()
+    hasher.update(buf)
+    return hasher.hexdigest()
+
+def verify_figure_hash(name, figure=None):
+    """
+    Verifies whether a figure has the same hash as the named hash in the current hash library.
+    If the hash library does not contain the specified name, the hash is added to the library.
+
+    Parameters
+    ----------
+    name : string
+        The identifier for the hash in the hash library
+    figure : matplotlib.figure.Figure
+        If None is specified, the current figure is used (as determined by matplotlib.pyplot.gcf())
+
+    Returns
+    -------
+    out : bool
+        False if the figure's hash does not match the named hash, otherwise True
+    """
+    if name not in hash_library:
+        hash_library[name] = hash_figure(figure)
+        return True
+    return hash_library[name] == hash_figure(figure)

sunpy/tests/helpers.py

@@ -7,7 +7,11 @@
 import pytest
 
 import astropy.units as u
+from astropy.utils.decorators import wraps
 import numpy as np
+import matplotlib.pyplot as plt
+
+from sunpy.tests import hash
 
 __all__ = ['skip_windows', 'skip_glymur', 'skip_ana', 'warnings_as_errors']
 
@@ -83,3 +87,30 @@ def assert_quantity_allclose(actual, desired, rtol=1.e-7, atol=0, err_msg='', ve
 
         np.testing.assert_allclose(actual, desired,
                                    rtol=rtol, atol=atol, err_msg=err_msg, verbose=verbose)
+
+
+def figure_test(test_function):
+    """
+    A decorator for a test that verifies the hash of the current figure or the returned figure,
+    with the name of the test function as the hash identifier in the library.
+
+    All such decorated tests are marked with `pytest.mark.figure` for convenient filtering.
+
+    Examples
+    --------
+    @figure_test
+    def test_simple_plot():
+        plt.plot([0,1])
+    """
+    @pytest.mark.figure
+    @wraps(test_function)
+    def wrapper(*args, **kwargs):
+        plt.figure()
+        name = "{0}.{1}".format(test_function.__module__, test_function.__name__)
+        figure_hash = hash.hash_figure(test_function(*args, **kwargs))
+        if name not in hash.hash_library:
+            hash.hash_library[name] = figure_hash
+            pytest.fail("Hash not present: {0}".format(name))
+        else:
+            assert hash.hash_library[name] == figure_hash
+    return wrapper