Merge pull request #2681 from dstansby/figure-diff

Output figure diff in test figure page
sunpy · Oct 10, 2018 · a68a754 · a68a754
2 parents c8f0a4e + 76b9882
commit a68a754
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 15 deletions.
diff --git a/changelog/2681.feature.rst b/changelog/2681.feature.rst
@@ -0,0 +1,2 @@
+Running the figure tests now creates a page showing the differences between
+the expected figures and the figures produced from running the tests.
diff --git a/sunpy/conftest.py b/sunpy/conftest.py
@@ -49,16 +49,16 @@ def figure_base_dir(request):
 
 def pytest_runtest_setup(item):
     """
-    pytest hook to skip all tests that have the mark 'online' if the
-    client is online (simply detected by checking whether http://www.google.com
-    can be requested).
+    pytest hook to skip all tests that have the mark 'remotedata' if the
+    pytest_remotedata plugin is not installed.
     """
     if isinstance(item, item.Function):
         if 'remote_data' in item.keywords and not HAVE_REMOTEDATA:
             pytest.skip("skipping remotedata tests as pytest-remotedata is not installed")
 
 
 def pytest_unconfigure(config):
+
     # If at least one figure test has been run, print result image directory
     if len(new_hash_library) > 0:
         # Write the new hash library in JSON
@@ -67,6 +67,18 @@ def pytest_unconfigure(config):
         with open(hashfile, 'w') as outfile:
             json.dump(new_hash_library, outfile, sort_keys=True, indent=4, separators=(',', ': '))
 
-        generate_figure_webpage()
+        """
+        Turn on internet when generating the figure comparison webpage.
+        """
+        if HAVE_REMOTEDATA:
+            from pytest_remotedata.disable_internet import turn_on_internet, turn_off_internet
+        else:
+            def turn_on_internet(): pass
+            def turn_off_internet(): pass
+
+        turn_on_internet()
+        generate_figure_webpage(new_hash_library)
+        turn_off_internet()
+
         print('All images from image tests can be found in {0}'.format(figure_base_dir))
         print("The corresponding hash library is {0}".format(hashfile))
diff --git a/sunpy/tests/helpers.py b/sunpy/tests/helpers.py
@@ -5,10 +5,12 @@
 import os
 import pathlib
 import platform
+import urllib
 import warnings
 
 import pytest
 import matplotlib.pyplot as plt
+from matplotlib.testing import compare
 
 from astropy.utils.decorators import wraps
 
@@ -145,6 +147,8 @@ def _patch_coverage(testdir, sourcedir):  # pragma: no cover
 
 
 html_intro = '''
+<!DOCTYPE html>
+<html>
 <head>
 <style>
 table, th, td {
@@ -158,25 +162,49 @@ def _patch_coverage(testdir, sourcedir):  # pragma: no cover
 
 <table>
   <tr>
-    <th>New image</th>
+    <th>Test Name</th>
     <th>Baseline image</th>
     <th>Diff</th>
+    <th>New image</th>
   </tr>
 '''
 
 
-def generate_figure_webpage():
+def _generate_fig_html(fname):
+    generated_image = figure_base_dir / (fname + '.png')
+
+    # Download baseline image
     baseline_url = 'https://raw.githubusercontent.com/sunpy/sunpy-figure-tests/master/figures/'
+    baseline_image_url = baseline_url + generated_image.name
+    baseline_image = figure_base_dir / (generated_image.stem + '_baseline' + generated_image.suffix)
+    baseline_image_exists = baseline_image.exists()
+    if not baseline_image_exists:
+        try:
+            urllib.request.urlretrieve(baseline_image_url, baseline_image)
+            baseline_image_exists = True
+        except urllib.error.HTTPError:
+            pass
+
+    # Create diff between baseline and generated image
+    diff_image = figure_base_dir / (generated_image.stem + '_diff' + generated_image.suffix)
+    if baseline_image_exists:
+        compare.save_diff_image(str(baseline_image), str(generated_image), str(diff_image))
+
+    html_block = ('<tr>'
+                  '<td>{}\n'.format(generated_image.stem) +
+                  '<td><img src="{}"></td>\n'.format(baseline_image.name) +
+                  '<td><img src="{}"></td>\n'.format(diff_image.name) +
+                  '<td><img src="{}"></td>\n'.format(generated_image.name) +
+                  '</tr>\n\n')
+    return html_block
+
+
+def generate_figure_webpage(hash_library):
     html_file = figure_base_dir / 'fig_comparison.html'
     with open(html_file, 'w') as f:
         f.write(html_intro)
-        for fname in figure_base_dir.iterdir():
-            if fname.suffix == '.png':
-                html_block = ('<tr>'
-                              '<td>{}\n'.format(fname.stem) +
-                              '<img src="{}"></td>\n'.format(fname.name) +
-                              '<td><img src="{}"></td>\n'.format(baseline_url + fname.name) +
-                              '<td></td>'
-                              '</tr>\n\n')
-                f.write(html_block)
+        for fname in hash_library:
+            f.write(_generate_fig_html(fname))
         f.write('</table>')
+        f.write('</body>')
+        f.write('</html>')