Merge pull request #76 from GeminiDRSoftware/fix/test_clean_up

Fix/test clean up

Jenkinsfile

@@ -101,13 +101,14 @@ pipeline {
                     environment {
                         MPLBACKEND = "agg"
                         PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
+                        DRAGONS_TEST_OUT = "$DRAGONS_TEST_OUT"
                     }
                     steps {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
                         sh '.jenkins/scripts/setup_agent.sh'
                         echo "Running tests with Python 3.7"
-                        sh 'tox -e py37-unit -v -- --junit-xml reports/unittests_results.xml'
+                        sh 'tox -e py37-unit -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/unittests_results.xml'
                         echo "Reportint coverage to CodeCov"
                         sh 'tox -e codecov -- -F unit'
                     }
@@ -123,66 +124,60 @@ pipeline {
             }
         }
 
-        stage('Integration tests') {
-            // when {
-            //     branch 'master'
-            // }
-            agent {
-                label "centos7"
-            }
+        stage('GMOS LS Tests') {
+            agent { label "master" }
             environment {
                 MPLBACKEND = "agg"
                 PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
+                DRAGONS_TEST_OUT = "$DRAGONS_TEST_OUT"
             }
             steps {
                 echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                 checkout scm
-                echo "${env.PATH}"
                 sh '.jenkins/scripts/setup_agent.sh'
-                echo "Integration tests"
-                sh 'tox -e py36-integ -v -- --junit-xml reports/integration_results.xml'
+                echo "Running tests"
+                sh 'tox -e py36-gmosls -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/unittests_results.xml'
                 echo "Reporting coverage"
-                sh 'tox -e codecov -- -F integration'
-            }
+                sh 'tox -e codecov -- -F gmosls'
+            }  // end steps
             post {
                 always {
+                    echo "Running 'archivePlots' from inside GmosArcTests"
+                    archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                     junit (
                         allowEmptyResults: true,
                         testResults: 'reports/*_results.xml'
                     )
-                }
-            }
-        }
+                }  // end always
+            }  // end post
+        }  // end stage
 
-        stage('GMOS LS Tests') {
-            agent {
-//                 label "centos7"
-                label "master"
-            }
+        stage('Integration tests') {
+            agent { label "centos7" }
             environment {
                 MPLBACKEND = "agg"
                 PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
+                DRAGONS_TEST_OUT = "$DRAGONS_TEST_OUT"
             }
             steps {
                 echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                 checkout scm
+                echo "${env.PATH}"
                 sh '.jenkins/scripts/setup_agent.sh'
-                echo "Running tests"
-                sh 'tox -e py36-gmosls -v -- --junit-xml reports/unittests_results.xml'
+                echo "Integration tests"
+                sh 'tox -e py36-integ -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/integration_results.xml'
                 echo "Reporting coverage"
-                sh 'tox -e codecov -- -F gmosls'
-            }  // end steps
+                sh 'tox -e codecov -- -F integration'
+            } // end steps
             post {
                 always {
-                    echo "Running 'archivePlots' from inside GmosArcTests"
-                    archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                     junit (
                         allowEmptyResults: true,
                         testResults: 'reports/*_results.xml'
                     )
-                }  // end always
-            }  // end post
-        }  // end stage
+                }
+            } // end post
+        } // end stage
 
     }
     post {

astrodata/testing.py

@@ -410,10 +410,11 @@ def path_to_refs(request, env_var='DRAGONS_TEST'):
 @pytest.fixture(scope='module')
 def path_to_outputs(request, tmp_path_factory):
     """
-    PyTest fixture that creates a temporary folder to save tests outputs.
+    PyTest fixture that creates a temporary folder to save tests outputs. You can
+    set the base directory by passing the ``--basetemp=mydir/`` argument to the
+    PyTest call (See [Pytest - Temporary Directories and Files][1]).
 
-    This output folder can be override via $DRAGONS_TEST_OUTPUTS environment
-    variable or via `--basetemp` argument.
+    [1]: https://docs.pytest.org/en/stable/tmpdir.html#temporary-directories-and-files
 
     Returns
     -------
@@ -425,18 +426,11 @@ def path_to_outputs(request, tmp_path_factory):
     IOError
         If output path does not exits.
     """
-    if os.getenv('DRAGONS_TEST_OUTPUTS'):
-        path = os.path.expanduser(os.getenv('DRAGONS_TEST_OUTPUTS'))
-        if not os.path.exists(path):
-            raise OSError(
-                "Could not access path stored in $DRAGONS_TEST_OUTPUTS: "
-                "{}\n Using current working directory".format(path))
-    else:
-        path = str(tmp_path_factory.getbasetemp())
+    path = tmp_path_factory.mktemp(basename="dragons-test-", numbered=True)
 
     module_path = request.module.__name__.split('.')
     module_path = [item for item in module_path if item not in "tests"]
-    path = os.path.join(path, *module_path)
+    path = os.path.join(str(path), *module_path)
     os.makedirs(path, exist_ok=True)
 
     return path

astrodata/tests/test_fits.py

@@ -152,6 +152,7 @@ def test_extver_remap(tmpdir):
         assert hdr['EXTVER'] == i + 1
 
     assert ad.extver_map() == {1: 0, 2: 1, 3: 2, 4: 3}
+    os.remove(testfile)
 
 
 @pytest.mark.dragons_remote_data
@@ -352,6 +353,7 @@ def test_paths(path_to_outputs, NIFS_DARK):
     assert ad.orig_filename == 'N20160727S0077.fits'
     ad.write()
     assert os.path.exists(testfile)
+    os.remove(testfile)
 
     testfile = os.path.join(path_to_outputs, 'temp2.fits')
     ad.write(testfile)
@@ -363,6 +365,7 @@ def test_paths(path_to_outputs, NIFS_DARK):
 
     ad.write(testfile, overwrite=True)
     assert os.path.exists(testfile)
+    os.remove(testfile)
 
     ad.path = None
     assert ad.filename is None
@@ -408,6 +411,7 @@ def test_can_make_and_write_ad_object(tmpdir):
     # Opens file again and tests data is same as above
     adnew = astrodata.open(testfile)
     assert np.array_equal(adnew[0].data, np.arange(10))
+    os.remove(testfile)
 
 
 def test_can_append_table_and_access_data(capsys, tmpdir):
@@ -431,6 +435,7 @@ def test_can_append_table_and_access_data(capsys, tmpdir):
     adnew = astrodata.open(testfile)
     assert adnew.exposed == {'BOB'}
     assert len(adnew.BOB) == 10
+    os.remove(testfile)
 
     del ad.BOB
     assert ad.tables == set()
@@ -583,6 +588,7 @@ def test_read_empty_file(tmpdir):
     assert len(ad) == 0
     assert ad.object() == 'M42'
     assert ad.instrument() == 'darkimager'
+    os.remove(testfile)
 
 
 def test_read_file(tmpdir):
@@ -593,6 +599,7 @@ def test_read_file(tmpdir):
     assert len(ad) == 0
     assert ad.object() == 'M42'
     assert ad.instrument() == 'darkimager'
+    os.remove(testfile)
 
 
 @pytest.mark.dragons_remote_data
@@ -916,6 +923,8 @@ def test_round_trip_gwcs(tmpdir):
     np.testing.assert_allclose(wcs1.invert(w, y), wcs2.invert(w, y),
                                rtol=1e-7, atol=0.)
 
+    os.remove(testfile)
+
 
 @pytest.mark.parametrize('dtype', ['int8', 'uint8', 'int16', 'uint16',
                                    'int32', 'uint32', 'int64', 'uint64'])
@@ -927,6 +936,7 @@ def test_uint_data(dtype, tmp_path):
     ad = astrodata.open(str(testfile))
     assert ad[0].data.dtype == data.dtype
     assert_array_equal(ad[0].data, data)
+    os.remove(testfile)
 
 
 if __name__ == '__main__':

astrodata/tests/test_tags.py

@@ -1,3 +1,4 @@
+import os
 import numpy as np
 import pytest
 from astropy.io import fits
@@ -72,7 +73,7 @@ def setup_module():
     factory.addClass(AstroDataMyInstrument)
 
 
-@pytest.fixture
+@pytest.fixture(scope='function')
 def testfile(tmpdir):
     hdr = fits.Header({
         'INSTRUME': 'MYINSTRUMENT',
@@ -90,7 +91,8 @@ def testfile(tmpdir):
     ad.append(tbl, name='MYCAT')
     filename = str(tmpdir.join('fakebias.fits'))
     ad.write(filename)
-    return filename
+    yield filename
+    os.remove(filename)
 
 
 def test_tags(testfile):

astrodata/tests/test_testing.py

@@ -33,6 +33,10 @@ def test_change_working_dir(change_working_dir):
 
     assert "astrodata/test_testing/outputs" not in os.getcwd()
 
+    dragons_basetemp = os.getenv("$DRAGONS_TEST_OUT")
+    if dragons_basetemp:
+        assert dragons_basetemp in os.getcwd()
+
 
 def test_download_from_archive_raises_ValueError_if_envvar_does_not_exists():
     with pytest.raises(ValueError):

conftest.py

@@ -27,6 +27,12 @@ def pytest_addoption(parser):
             default=False,
             help="Plot results of each test after running them."
         )
+        parser.addoption(
+            "--keep-data",
+            action="store_true",
+            default=False,
+            help="Keep intermediate data (e.g. pre-stack data)."
+        )
     # This file is imported several times and might bring conflict
     except ValueError:
         pass

geminidr/f2/recipes/tests/test_reduce.py

@@ -83,7 +83,7 @@
 
 
 @pytest.mark.skip(reason='Test needs refactoring')
-@pytest.mark.integtest
+@pytest.mark.integration_test
 def test_reduce_image(change_working_dir):
     with change_working_dir():
         calib_files = []

geminidr/gmos/recipes/qa/tests/test_flat_image.py

@@ -2,8 +2,10 @@
 """
 Tests the GMOS Image Bias reduction for QA mode.
 """
+import glob
 import os
 import pytest
+import shutil
 
 import astrodata
 import gemini_instruments
@@ -21,9 +23,7 @@
 ]
 
 
-# ToDo - @bquint - Perform clean up after running tests
-@pytest.mark.skip("Using too much space - enable clean up")
-@pytest.mark.integtest
+@pytest.mark.integration_test
 @pytest.mark.parametrize("master_bias, flat_fnames", test_data)
 def test_make_processed_flat(
         change_working_dir, flat_fnames, master_bias, path_to_inputs):
@@ -53,6 +53,10 @@ def test_make_processed_flat(
         r.ucals = normalize_ucals(r.files, calibration_files)
         r.runr()
 
+        # Delete files that won't be used
+        shutil.rmtree('calibrations/')
+        [os.remove(f) for f in glob.glob('*_forStack.fits')]
+
         ad = astrodata.open(r.output_filenames[0])
 
     for ext in ad:

geminidr/gmos/recipes/ql/tests/test_flat_ls_spect.py

@@ -2,6 +2,7 @@
 import os
 import numpy as np
 import pytest
+import shutil
 
 import astrodata
 # noinspection PyUnresolvedReferences
@@ -34,10 +35,10 @@
 
 
 # -- Tests --------------------------------------------------------------------
-# ToDo - @bquint - Perform clean up after running tests
-@pytest.mark.skip("Using too much space - enable clean up")
+@pytest.mark.dragons_remote_data
 @pytest.mark.gmosls
 @pytest.mark.parametrize("processed_flat", datasets, indirect=True)
+@pytest.mark.preprocessed_data
 def test_processed_flat_has_median_around_one(processed_flat):
     """
     Tests if the processed flat contains values around one.
@@ -52,10 +53,10 @@ def test_processed_flat_has_median_around_one(processed_flat):
         np.testing.assert_almost_equal(np.median(data.ravel()), 1.0, decimal=3)
 
 
-# ToDo - @bquint - Perform clean up after running tests
-@pytest.mark.skip("Using too much space - enable clean up")
+@pytest.mark.dragons_remote_data
 @pytest.mark.gmosls
 @pytest.mark.parametrize("processed_flat", datasets, indirect=True)
+@pytest.mark.preprocessed_data
 def test_processed_flat_has_small_std(processed_flat):
     """
     Tests if the processed flat has a small standard deviation.
@@ -70,10 +71,10 @@ def test_processed_flat_has_small_std(processed_flat):
         np.testing.assert_array_less(np.std(data.ravel()), 0.1)
 
 
-# ToDo - @bquint - Perform clean up after running tests
-@pytest.mark.skip("Using too much space - enable clean up")
+@pytest.mark.dragons_remote_data
 @pytest.mark.gmosls
 @pytest.mark.parametrize("processed_flat", datasets, indirect=True)
+@pytest.mark.preprocessed_data
 def test_regression_processed_flat(processed_flat, ref_ad_factory):
     """
     Tests if the processed flat contains values around one.
@@ -133,6 +134,9 @@ def processed_flat(change_working_dir, path_to_inputs, request):
         reduce.ucals = normalize_ucals(reduce.files, calibration_files)
         reduce.runr()
 
+        # Clean up duplicated files
+        shutil.rmtree('calibrations/')
+
         _processed_flat_filename = reduce.output_filenames.pop()
         _processed_flat = astrodata.open(_processed_flat_filename)
 
@@ -204,6 +208,8 @@ def create_master_bias_for_tests():
         bias_reduce = Reduce()
         bias_reduce.files.extend(bias_paths)
         bias_reduce.runr()
+
+        shutil.rmtree("calibrations/")
 
 
 if __name__ == '__main__':