[WIP] Adding a nightly conda build for image comparison

.circleci/config.yml

@@ -11,7 +11,24 @@ pip-run: &pip-install
     . venv/bin/activate
     pip install -q -r requirements/docs.txt
 
+conda-run: &conda-install
+  name: Install conda environment from env file
+  command: |
+    wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
+    export PATH=/home/circleci/miniconda2/bin:$PATH
+    chmod +x miniconda.sh && ./miniconda.sh -b
+    conda env create --file sunpy/tests/figure_tests_env_3.5.yml
+    source activate sunpy-figure-tests-3.5
+    python --version
+
+figure-run: &figure-tests
+  name: Run figure tests
+  command: |
+    python setup.py test --figure-only
+    python sunpy/tests/circle_image_tests.py
+
 version: 2
+
 jobs:
   egg-info-36:
     docker:
@@ -58,6 +75,16 @@ jobs:
           paths:
             - ~/sunpy/data/sample_data
 
+  nightly-35:
+    docker:
+      - image: circleci/python:3.5
+    steps:
+      - run: export ENV_FILE="figure_tests_env_3.5.yml"
+      - run: export ENV_NAME="sunpy-figure-tests-3.5"
+      - checkout
+      - run: *conda-install
+      - run: *figure-tests
+
 
 workflows:
   version: 2
@@ -68,9 +95,9 @@ workflows:
       - egg-info-35
       - egg-info-27
 
-  test-documentation:
+  testing-nightly:
     jobs:
-      - html-docs
+      - nightly-35
 
 
 notify:

.gitignore

@@ -75,3 +75,6 @@ $RECYCLE.BIN/
 .hypothesis/
 .pytest_cache/
 sunpydata.sqlite
+
+result_images*
+.pytest_cache

sunpy/conftest.py

@@ -14,7 +14,7 @@
     matplotlib.use('Agg')
 
 from sunpy.tests.hash import HASH_LIBRARY_NAME
-from sunpy.tests.helpers import new_hash_library, figure_test_pngfiles
+from sunpy.tests.helpers import new_hash_library, test_fig_dir
 from sunpy.extern import six
 
 import pytest
@@ -47,17 +47,10 @@ def pytest_runtest_setup(item):
 
 
 def pytest_unconfigure(config):
-    if len(figure_test_pngfiles) > 0:
-        tempdir = tempfile.mkdtemp(suffix="_figures")
-
-        # Rename each PNG with the name of the corresponding test
-        for test_name in figure_test_pngfiles:
-            os.rename(figure_test_pngfiles[test_name], os.path.join(tempdir, test_name + '.png'))
-
+    if len(new_hash_library) > 0:
         # Write the new hash library in JSON
-        hashfile = os.path.join(tempdir, HASH_LIBRARY_NAME)
+        hashfile = os.path.join(test_fig_dir, HASH_LIBRARY_NAME)
         with open(hashfile, 'w') as outfile:
             json.dump(new_hash_library, outfile, sort_keys=True, indent=4, separators=(',', ': '))
 
-        print('All test files for figure hashes can be found in {0}'.format(tempdir))
         print("The corresponding hash library is {0}".format(hashfile))

sunpy/tests/circle_image_tests.py

@@ -0,0 +1,14 @@
+"""
+A script to check the image test results on CircleCI, and upload
+any new images to the sunpy-figure-tests repository.
+"""
+import os
+
+image_directory = None
+for f in os.listdir('.'):
+    if 'result_images' in f:
+        image_directory = f
+        print(image_directory)
+
+if image_directory is None:
+    raise RuntimeError("Could not find directory with saved images")

sunpy/tests/figure_tests_env_3.5.yml

@@ -91,12 +91,7 @@ dependencies:
   - xorg-libxdmcp=1.1.2=3
   - xz=5.2.3=0
   - zlib=1.2.11=0
-  - intel-openmp=2018.0.0=hc7b2577_8
-  - libgcc=7.2.0=h69d50b8_2
-  - libgcc-ng=7.2.0=h7cc24e2_2
   - libgfortran=3.0.0=1
-  - libstdcxx-ng=7.2.0=h7a57d05_2
-  - mkl=2018.0.1=h19d6760_4
   - pip:
     - dask==0.16.1
     - psutil==5.4.3

sunpy/tests/hash.py

@@ -46,11 +46,18 @@ def hash_figure(figure=None, out_stream=None):
 
     figure.savefig(imgdata, format='png')
 
-    imgdata.seek(0)
-    buf = imgdata.read()
+    out = _hash_file(imgdata)
     if out_stream is None:
         imgdata.close()
+    return out
 
+
+def _hash_file(in_stream):
+    """
+    Hashes an already opened file
+    """
+    in_stream.seek(0)
+    buf = in_stream.read()
     hasher = hashlib.sha256()
     hasher.update(buf)
     return hasher.hexdigest()

sunpy/tests/helpers.py

@@ -7,6 +7,7 @@
 import tempfile
 import platform
 import os
+import datetime
 
 import pytest
 import numpy as np
@@ -40,31 +41,30 @@
     SKIP_ANA = False
 
 skip_windows = pytest.mark.skipif(platform.system() == 'Windows', reason="Windows")
-
 skip_glymur = pytest.mark.skipif(SKIP_GLYMUR, reason="Glymur can not be imported")
-
 skip_ana = pytest.mark.skipif(SKIP_ANA, reason="ANA is not available")
 
-
 @pytest.fixture
 def warnings_as_errors(request):
     warnings.simplefilter('error')
 
     request.addfinalizer(lambda *args: warnings.resetwarnings())
 
-new_hash_library = {}
 
-figure_test_pngfiles = {}
+hash_library = hash.hash_library
+new_hash_library = {}
+test_fig_dir = 'result_images_{:%H%M%S}'.format(datetime.datetime.now())
 
 
 def figure_test(test_function):
     """
     A decorator for a test that verifies the hash of the current figure or the returned figure,
     with the name of the test function as the hash identifier in the library.
-    A PNG is also created with a temporary filename, with the lookup stored in the
-    `figure_test_pngfiles` dictionary.
+    A PNG is also created in the 'result_image' director, which is created
+    on the current path.
 
-    All such decorated tests are marked with `pytest.mark.figure` for convenient filtering.
+    All such decorated tests are marked with `pytest.mark.figure` for
+    convenient filtering.
 
     Examples
     --------
@@ -77,16 +77,33 @@ def test_simple_plot():
     def wrapper(*args, **kwargs):
         if not os.path.exists(hash.HASH_LIBRARY_FILE):
             pytest.xfail('Could not find a figure hash library at {}'.format(hash.HASH_LIBRARY_FILE))
+
+        name = "{0}.{1}".format(test_function.__module__,
+                                test_function.__name__)
+        # Run the test function and get the figure
         plt.figure()
-        name = "{0}.{1}".format(test_function.__module__, test_function.__name__)
-        pngfile = tempfile.NamedTemporaryFile(delete=False)
-        figure_hash = hash.hash_figure(test_function(*args, **kwargs), out_stream=pngfile)
-        figure_test_pngfiles[name] = pngfile.name
-        pngfile.close()
+        fig = test_function(*args, **kwargs)
+        if fig is None:
+            fig = plt.gcf()
+
+        # Save the image that was generated
+        if not os.path.exists(test_fig_dir):
+            os.mkdir(test_fig_dir)
+        result_image_loc = os.path.join(test_fig_dir, '{}.png'.format(name))
+        plt.savefig(result_image_loc)
+        plt.close()
+
+        # Create hash
+        imgdata = open(result_image_loc, "rb")
+        figure_hash = hash._hash_file(imgdata)
+        imgdata.close()
+
         new_hash_library[name] = figure_hash
-        if name not in hash.hash_library:
+        if name not in hash_library:
             pytest.fail("Hash not present: {0}".format(name))
-        else:
-            assert hash.hash_library[name] == figure_hash
-        plt.close()
+
+        if hash_library[name] != figure_hash:
+            raise RuntimeError('Figure hash does not match expected hash.\n'
+                               'New image generated and placed at {}'.format(result_image_loc))
+
     return wrapper